beetle 0.2.6 → 0.2.9

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,11 @@
 = Release Notes
 
+== Version 0.2.9
+
+* Beetle::Client now raises an exception when it fails to publish a message to at least 1 RabbitMQ server
+* Subscribers are now stopped cleanly to avoid 'closed abruptly' messages in the RabbitMQ server log
+* Added send and receive timeouts on the socket and use system_timer for ruby side timeouts
+
 == Version 0.2.6
 
 * Set dependency on ActiveSupport to 2.3.x since it ain't compatible to version 3.x yet

data/beetle.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = "beetle"
-  s.version = "0.2.6"
+  s.version = "0.2.9"
 
   s.required_rubygems_version = ">= 1.3.1"
   s.authors            = ["Stefan Kaes", "Pascal Friederich", "Ali Jelveh", "Sebastian Roebke"]
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
   s.specification_version = 3
   s.add_runtime_dependency("uuid4r",                [">= 0.1.1"])
   s.add_runtime_dependency("bunny",                 [">= 0.6.0"])
-  s.add_runtime_dependency("redis",                 [">= 2.0.4"])
+  s.add_runtime_dependency("redis",                 ["= 2.0.4"])
   s.add_runtime_dependency("amqp",                  [">= 0.6.7"])
   s.add_runtime_dependency("activesupport",         ["~> 2.3.4"])
   s.add_runtime_dependency("daemons",               [">= 1.0.10"])

data/examples/attempts.rb

@@ -14,15 +14,15 @@ require File.expand_path("../lib/beetle", File.dirname(__FILE__))
 Beetle.config.logger.level = Logger::INFO
 
 # setup client
-client = Beetle::Client.new
-client.register_queue(:test)
-client.register_message(:test)
+$client = Beetle::Client.new
+$client.register_queue(:test)
+$client.register_message(:test)
 
 # purge the test queue
-client.purge(:test)
+$client.purge(:test)
 
 # empty the dedup store
-client.deduplication_store.flushdb
+$client.deduplication_store.flushdb
 
 # we're starting with 0 exceptions and expect our handler to process the message until the exception count has reached 10
 $exceptions = 0
@@ -32,35 +32,35 @@ $max_exceptions = 10
 # in this example we've not only overwritten the process method but also the
 # error and failure methods of the handler baseclass
 class Handler < Beetle::Handler
-  
+
   # called when the handler receives the message - fail everytime
   def process
     raise "failed #{$exceptions += 1} times"
   end
-  
+
   # called when handler process raised an exception
   def error(exception)
     logger.info "execution failed: #{exception}"
   end
-  
+
   # called when the handler has finally failed
   # we're stopping the event loop so this script stops after that
   def failure(result)
     super
-    EM.stop_event_loop
+    $client.stop_listening
   end
 end
 
 # register our handler to the message, configure it to our max_exceptions limit, we configure a delay of 0 to have it not wait before retrying
-client.register_handler(:test, Handler, :exceptions => $max_exceptions, :delay => 0)
+$client.register_handler(:test, Handler, :exceptions => $max_exceptions, :delay => 0)
 
 # publish a our test message
-client.publish(:test, "snafu")
+$client.publish(:test, "snafu")
 
 # and start our listening loop...
-client.listen
+$client.listen
 
 # error handling, if everything went right this shouldn't happen.
 if $exceptions != $max_exceptions + 1
   raise "something is fishy. Failed #{$exceptions} times"
-end
+end

data/lib/beetle.rb

@@ -1,3 +1,4 @@
+$:.unshift(File.expand_path('..', __FILE__))
 require 'amqp'
 require 'mq'
 require 'bunny'
@@ -17,6 +18,8 @@ module Beetle
   class UnknownQueue < Error; end
   # raised when no redis master server can be found
   class NoRedisMaster < Error; end
+  # raise when no message could be sent by the publisher
+  class NoMessageSent < Error; end
 
   # AMQP options for exchange creation
   EXCHANGE_CREATION_KEYS  = [:auto_delete, :durable, :internal, :nowait, :passive]
@@ -56,3 +59,5 @@ module Beetle
 
   Timer = RUBY_VERSION < "1.9" ? SystemTimer : Timeout
 end
+
+require 'ext/qrack/client'

data/lib/beetle/client.rb

@@ -24,6 +24,9 @@ module Beetle
     # the AMQP servers available for publishing
     attr_reader :servers
 
+    # additional AMQP servers available for subscribing. useful for migration scenarios.
+    attr_reader :additional_subscription_servers
+
     # an options hash for the configured exchanges
     attr_reader :exchanges
 
@@ -46,6 +49,7 @@ module Beetle
     def initialize(config = Beetle.config)
       @config  = config
       @servers = config.servers.split(/ *, */)
+      @additional_subscription_servers = config.additional_subscription_servers.split(/ *, */)
       @exchanges = {}
       @queues = {}
       @messages = {}

data/lib/beetle/configuration.rb

@@ -38,6 +38,8 @@ module Beetle
 
     # list of amqp servers to use (defaults to <tt>"localhost:5672"</tt>)
     attr_accessor :servers
+    # list of additional amqp servers to use for subscribers (defaults to <tt>""</tt>)
+    attr_accessor :additional_subscription_servers
     # the virtual host to use on the AMQP servers (defaults to <tt>"/"</tt>)
     attr_accessor :vhost
     # the AMQP user to use when connecting to the AMQP servers (defaults to <tt>"guest"</tt>)
@@ -63,6 +65,7 @@ module Beetle
       self.redis_configuration_client_ids = ""
 
       self.servers = "localhost:5672"
+      self.additional_subscription_servers = ""
       self.vhost = "/"
       self.user = "guest"
       self.password = "guest"
@@ -77,13 +80,14 @@ module Beetle
     end
 
     def logger
-      @logger ||= begin
-        l = Logger.new(log_file)
-        l.formatter = Logger::Formatter.new
-        l.level = Logger::INFO
-        l.datetime_format = "%Y-%m-%d %H:%M:%S"
-        l
-      end
+      @logger ||=
+        begin
+          l = Logger.new(log_file)
+          l.formatter = Logger::Formatter.new
+          l.level = Logger::INFO
+          l.datetime_format = "%Y-%m-%d %H:%M:%S"
+          l
+        end
     end
 
     private
@@ -93,6 +97,7 @@ module Beetle
         send("#{key}=", value)
       end
     rescue Exception
+      Beetle::reraise_expectation_errors!
       logger.error "Error loading beetle config file '#{config_file}': #{$!}"
       raise
     end

data/lib/beetle/message.rb

@@ -80,6 +80,7 @@ module Beetle
       @flags = headers[:flags].to_i
       @expires_at = headers[:expires_at].to_i
     rescue Exception => @exception
+      Beetle::reraise_expectation_errors!
       logger.error "Could not decode message. #{self.inspect}"
     end
 

data/lib/beetle/publisher.rb

@@ -50,6 +50,7 @@ module Beetle
         tries -= 1
         retry if tries > 0
         logger.error "Beetle: message could not be delivered: #{message_name}"
+        raise NoMessageSent.new
       end
       published
     end
@@ -79,9 +80,11 @@ module Beetle
       case published.size
       when 0
         logger.error "Beetle: message could not be delivered: #{message_name}"
+        raise NoMessageSent.new
       when 1
         logger.warn "Beetle: failed to send message redundantly"
       end
+
       published.size
     end
 

data/lib/beetle/subscriber.rb

@@ -5,6 +5,7 @@ module Beetle
     # create a new subscriber instance
     def initialize(client, options = {}) #:nodoc:
       super
+      @servers.concat @client.additional_subscription_servers
       @handlers = {}
       @amqp_connections = {}
       @mqs = {}
@@ -30,9 +31,14 @@ module Beetle
       end
     end
 
-    # stops the eventmachine loop
+    # closes all AMQP connections and stops the eventmachine loop
     def stop! #:nodoc:
-      EM.stop_event_loop
+      if @amqp_connections.empty?
+        EM.stop_event_loop
+      else
+        server, connection = @amqp_connections.shift
+        connection.close { stop! }
+      end
     end
 
     # register handler for the given queues (see Client#register_handler)

data/lib/ext/qrack/client.rb

@@ -0,0 +1,27 @@
+require 'qrack/client'
+
+
+module Qrack
+  class Client
+    # overwrite the timeout method so that SystemTimer is used
+    # instead the standard timeout.rb: http://ph7spot.com/musings/system-timer
+    delegate :timeout, :to => Beetle::Timer
+
+    def socket_with_reliable_timeout
+      socket_without_reliable_timeout
+
+      secs   = Integer(CONNECT_TIMEOUT)
+      usecs  = Integer((CONNECT_TIMEOUT - secs) * 1_000_000)
+      optval = [secs, usecs].pack("l_2")
+      
+      begin
+        @socket.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
+        @socket.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
+      rescue Errno::ENOPROTOOPT
+      end
+      @socket
+    end
+    alias_method_chain :socket, :reliable_timeout
+
+  end
+end

data/test/beetle/client_test.rb

@@ -11,6 +11,10 @@ module Beetle
       assert_equal ["localhost:5672"], @client.servers
     end
 
+    test "should have no additional subscription servers" do
+      assert_equal [], @client.additional_subscription_servers
+    end
+
     test "should have no exchanges" do
       assert @client.exchanges.empty?
     end

data/test/beetle/configuration_test.rb

@@ -12,7 +12,7 @@ module Beetle
       config.config_file = "some/path/to/a/file"
       assert_equal new_value, config.gc_threshold
     end
-    
+
     test "should log to STDOUT if no log_file given" do
       config = Configuration.new
       Logger.expects(:new).with(STDOUT).returns(stub_everything)
@@ -27,4 +27,4 @@ module Beetle
       config.logger
     end
   end
-end
+end

data/test/beetle/deduplication_store_test.rb

@@ -95,6 +95,8 @@ module Beetle
       redis1.expects(:get).with("foo:x").raises("disconnected").in_sequence(s)
       @store.expects(:redis).returns(redis2).in_sequence(s)
       redis2.expects(:get).with("foo:x").returns("42").in_sequence(s)
+      @store.logger.expects(:info)
+      @store.logger.expects(:error)
       assert_equal("42", @store.get("foo", "x"))
     end
 
@@ -103,6 +105,8 @@ module Beetle
       @store.stubs(:redis).returns(redis1)
       redis1.stubs(:get).with("foo:x").raises("disconnected")
       @store.stubs(:sleep)
+      @store.logger.stubs(:info)
+      @store.logger.stubs(:error)
       assert_raises(NoRedisMaster) { @store.get("foo", "x") }
     end
   end

data/test/beetle/ext_test.rb

@@ -0,0 +1,26 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+
+class QrackClientExtTest < Test::Unit::TestCase
+  def setup
+    Qrack::Client.any_instance.stubs(:create_channel).returns(nil)
+    @client = Qrack::Client.new
+  end
+
+
+  test "should use system-timer for reliable timeouts" do
+    Beetle::Timer.expects(:timeout)
+    @client.send :timeout, 1, 1 do
+    end
+  end
+
+  test "should set send/receive timeouts on the socket" do
+    socket_mock = mock("socket")
+    @client.instance_variable_set(:@socket, socket_mock)
+    @client.stubs(:socket_without_reliable_timeout)
+
+    socket_mock.expects(:setsockopt).with(Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, anything)
+    socket_mock.expects(:setsockopt).with(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, anything)
+    @client.send(:socket)
+  end
+end

data/test/beetle/message_test.rb

@@ -525,7 +525,7 @@ module Beetle
     test "processing a message catches internal exceptions risen by process_internal and returns an internal error" do
       header = header_with_params({})
       message = Message.new("somequeue", header, 'foo', :store => @store)
-      message.expects(:process_internal).raises(Exception.new)
+      message.expects(:process_internal).raises(Exception.new("this is expected"))
       handler = Handler.new
       handler.expects(:process_exception).never
       handler.expects(:process_failure).never

data/test/beetle/publisher_test.rb

@@ -76,13 +76,16 @@ module Beetle
     end
 
     test "publishing should fail over to the next server" do
-      failover = sequence('failover')
-      @pub.expects(:select_next_server).in_sequence(failover)
-      e = mock("exchange")
-      @pub.expects(:exchange).with("mama-exchange").returns(e).in_sequence(failover)
-      e.expects(:publish).raises(Bunny::ConnectionError).in_sequence(failover)
-      @pub.expects(:stop!).in_sequence(failover)
-      @pub.expects(:mark_server_dead).in_sequence(failover)
+      @pub.servers << "localhost:3333"
+      raising_exchange = mock("raising exchange")
+      nice_exchange = mock("nice exchange")
+      @pub.stubs(:exchange).with("mama-exchange").returns(raising_exchange).then.returns(nice_exchange)
+
+      raising_exchange.expects(:publish).raises(Bunny::ConnectionError)
+      nice_exchange.expects(:publish)
+      @pub.expects(:set_current_server).twice
+      @pub.expects(:stop!).once
+      @pub.expects(:mark_server_dead).once
       @pub.publish_with_failover("mama-exchange", "mama", @data, @opts)
     end
 
@@ -114,7 +117,7 @@ module Beetle
       assert_equal 1, @pub.publish_with_redundancy("mama-exchange", "mama", @data, @opts)
     end
 
-    test "redundant publishing should return 0 if the message was published to no server" do
+    test "redundant publishing should raise an exception if the message was published to no server" do
       redundant = sequence("redundant")
       @pub.servers = ["someserver", "someotherserver"]
       @pub.server = "someserver"
@@ -125,7 +128,9 @@ module Beetle
       @pub.expects(:exchange).with("mama-exchange").returns(e).in_sequence(redundant)
       e.expects(:publish).raises(Bunny::ConnectionError).in_sequence(redundant)
 
-      assert_equal 0, @pub.publish_with_redundancy("mama-exchange", "mama", @data, @opts)
+      assert_raises Beetle::NoMessageSent do
+        @pub.publish_with_redundancy("mama-exchange", "mama", @data, @opts)
+      end
     end
 
     test "redundant publishing should fallback to failover publishing if less than one server is available" do

data/test/beetle/redis_configuration_client_test.rb

@@ -65,13 +65,17 @@ module Beetle
 
     test "should clear redis master file if redis from master file is slave" do
       @client.stubs(:redis_master_from_master_file).returns(stub(:master? => false))
+      Beetle::Client.any_instance.stubs(:publish)
       @client.expects(:clear_redis_master_file)
+      @client.expects(:client_started!)
       @client.start
     end
 
     test "should clear redis master file if redis from master file is not available" do
       @client.stubs(:redis_master_from_master_file).returns(nil)
+      Beetle::Client.any_instance.stubs(:publish)
       @client.expects(:clear_redis_master_file)
+      @client.expects(:client_started!)
       @client.start
     end
 

data/test/beetle/subscriber_test.rb

@@ -51,6 +51,19 @@ module Beetle
 
   end
 
+  class AdditionalSubscriptionServersTest < Test::Unit::TestCase
+    def setup
+      @config = Configuration.new
+      @config.additional_subscription_servers = "localhost:1234"
+      @client = Client.new(@config)
+      @sub = @client.send(:subscriber)
+    end
+
+    test "subscribers server list should contain addtional subcription hosts" do
+      assert_equal ["localhost:5672", "localhost:1234"], @sub.servers
+    end
+  end
+
   class SubscriberQueueManagementTest < Test::Unit::TestCase
     def setup
       @client = Client.new
@@ -143,7 +156,7 @@ module Beetle
 
     test "exceptions raised from message processing should be ignored" do
       header = header_with_params({})
-      Message.any_instance.expects(:process).raises(Exception.new)
+      Message.any_instance.expects(:process).raises(Exception.new("don't worry"))
       assert_nothing_raised { @callback.call(header, 'foo') }
     end
 
@@ -214,12 +227,12 @@ module Beetle
       proc = lambda do |m|
         block_called = true
         assert_equal header, m.header
-        assert_equal "data", m.data
+        assert_equal "foo", m.data
         assert_equal server, m.server
       end
       @sub.register_handler("some_queue", &proc)
       q = mock("QUEUE")
-      q.expects(:subscribe).with({:ack => true, :key => "#"}).yields(header, 'foo')
+      q.expects(:subscribe).with({:ack => true, :key => "#"}).yields(header, "foo")
       @sub.expects(:queues).returns({"some_queue" => q})
       @sub.send(:subscribe, "some_queue")
       assert block_called

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: beetle
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 5
   prerelease: false
   segments: 
   - 0
   - 2
-  - 6
-  version: 0.2.6
+  - 9
+  version: 0.2.9
 platform: ruby
 authors: 
 - Stefan Kaes
@@ -18,7 +18,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-08 00:00:00 +02:00
+date: 2010-10-27 00:00:00 +02:00
 default_executable: beetle
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -59,7 +59,7 @@ dependencies:
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - "="
       - !ruby/object:Gem::Version 
         hash: 7
         segments: 
@@ -217,6 +217,7 @@ files:
 - lib/beetle/redis_server_info.rb
 - lib/beetle/subscriber.rb
 - lib/beetle.rb
+- lib/ext/qrack/client.rb
 - features/README.rdoc
 - features/redis_auto_failover.feature
 - features/step_definitions/redis_auto_failover_steps.rb
@@ -240,6 +241,7 @@ files:
 - test/beetle/client_test.rb
 - test/beetle/configuration_test.rb
 - test/beetle/deduplication_store_test.rb
+- test/beetle/ext_test.rb
 - test/beetle/handler_test.rb
 - test/beetle/message_test.rb
 - test/beetle/publisher_test.rb
@@ -293,6 +295,7 @@ test_files:
 - test/beetle/client_test.rb
 - test/beetle/configuration_test.rb
 - test/beetle/deduplication_store_test.rb
+- test/beetle/ext_test.rb
 - test/beetle/handler_test.rb
 - test/beetle/message_test.rb
 - test/beetle/publisher_test.rb