beetle 0.2.9.6 → 0.2.9.9

data/RELEASE_NOTES.rdoc

@@ -1,10 +1,24 @@
 = Release Notes
 
+== Version 0.2.9.8
+
+* since version 2.0, RabbitMQ supports Basic.reject(:requeue => true). we use it now too,
+  because it enhances performance of message processors. this means of course, you can
+  only use beetle gem versions >= 0.2.9.8 if your rabbitmq brokers are at least version 2.0.
+* publishing timeout defaults to 0 to avoid extreme message loss in some cases
+
+
+== Version 0.2.9.7
+
+* use new bunny_ext gem and allow specification of global publishing timeouts
+* registering a message now automatically registers the corresponding exchange
+* don't try to bind queues for an exchange hich has no queue
+* ruby 1.9.2 compatibility fixes
+
 == Version 0.2.9
 
 * Beetle::Client now raises an exception when it fails to publish a message to at least 1 RabbitMQ server
 * Subscribers are now stopped cleanly to avoid 'closed abruptly' messages in the RabbitMQ server log
-* Added send and receive timeouts on the socket and use system_timer for ruby side timeouts
 
 == Version 0.2.6
 

data/beetle.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = "beetle"
-  s.version = "0.2.9.6"
+  s.version = "0.2.9.9"
 
   s.required_rubygems_version = ">= 1.3.1"
   s.authors            = ["Stefan Kaes", "Pascal Friederich", "Ali Jelveh", "Sebastian Roebke"]
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
   s.specification_version = 3
   s.add_runtime_dependency("uuid4r",                [">= 0.1.1"])
   s.add_runtime_dependency("bunny",                 ["= 0.6.0"])
-  s.add_runtime_dependency("bunny-ext",             [">= 0.6.2"])
+  s.add_runtime_dependency("bunny-ext",             [">= 0.6.5"])
   s.add_runtime_dependency("redis",                 ["= 2.0.4"])
   s.add_runtime_dependency("amqp",                  [">= 0.6.7"])
   s.add_runtime_dependency("activesupport",         ["~> 2.3.4"])

data/lib/beetle/configuration.rb

@@ -47,7 +47,8 @@ module Beetle
     # the password to use when connectiong to the AMQP servers (defaults to <tt>"guest"</tt>)
     attr_accessor :password
 
-    # the socket timeout in seconds for message publishing (defaults to <tt>10.0</tt>)
+    # the socket timeout in seconds for message publishing (defaults to <tt>0</tt>).
+    # consider this a highly experimental feature for now.
     attr_accessor :publishing_timeout
 
     # external config file (defaults to <tt>no file</tt>)
@@ -73,7 +74,7 @@ module Beetle
       self.user = "guest"
       self.password = "guest"
 
-      self.publishing_timeout = 5.0
+      self.publishing_timeout = 0
 
       self.log_file = STDOUT
     end

data/lib/beetle/publisher.rb

@@ -44,8 +44,8 @@ module Beetle
         exchange(exchange_name).publish(data, opts)
         logger.debug "Beetle: message sent!"
         published = 1
-      rescue *bunny_exceptions
-        stop!
+      rescue *bunny_exceptions => e
+        stop!(e)
         tries -= 1
         # retry same server on receiving the first exception for it (might have been a normal restart)
         # in this case you'll see either a broken pipe or a forced connection shutdown error
@@ -76,8 +76,8 @@ module Beetle
           exchange(exchange_name).publish(data, opts)
           published << @server
           logger.debug "Beetle: message sent (#{published})!"
-        rescue *bunny_exceptions
-          stop!
+        rescue *bunny_exceptions => e
+          stop!(e)
           retry if (tries += 1) == 1
           mark_server_dead
         end
@@ -120,8 +120,8 @@ module Beetle
           status = msg[:header].properties[:headers][:status]
         end
         logger.debug "Beetle: rpc complete!"
-      rescue *bunny_exceptions
-        stop!
+      rescue *bunny_exceptions => e
+        stop!(e)
         mark_server_dead
         tries -= 1
         retry if tries > 0
@@ -200,10 +200,17 @@ module Beetle
       queue
     end
 
-    def stop!
+    def stop!(exception=nil)
       begin
-        bunny.stop
-      rescue Exception
+        Beetle::Timer.timeout(1) do
+          if exception
+            bunny.__send__ :close_socket
+          else
+            bunny.stop
+          end
+        end
+      rescue Exception => e
+        logger.error "Beetle: error closing down bunny #{e}"
         Beetle::reraise_expectation_errors!
       ensure
         @bunnies[@server] = nil

data/lib/beetle/r_c.rb

@@ -4,7 +4,7 @@ module Beetle
     # message processing result return codes
     class ReturnCode
       def initialize(*args)
-        @recover = args.delete :recover
+        @reject = args.delete :reject
         @failure = args.delete :failure
         @name = args.first
       end
@@ -13,8 +13,8 @@ module Beetle
         @name.blank? ? super : "Beetle::RC::#{@name}"
       end
 
-      def recover?
-        @recover
+      def reject?
+        @reject
       end
 
       def failure?
@@ -30,11 +30,11 @@ module Beetle
     rc :Ancient
     rc :AttemptsLimitReached, :failure
     rc :ExceptionsLimitReached, :failure
-    rc :Delayed, :recover
-    rc :HandlerCrash, :recover
-    rc :HandlerNotYetTimedOut, :recover
-    rc :MutexLocked, :recover
-    rc :InternalError, :recover
+    rc :Delayed, :reject
+    rc :HandlerCrash, :reject
+    rc :HandlerNotYetTimedOut, :reject
+    rc :MutexLocked, :reject
+    rc :InternalError, :reject
     rc :DecodingError, :failure
 
   end

data/lib/beetle/redis_configuration_server.rb

@@ -78,7 +78,7 @@ module Beetle
     def client_started(payload)
       id = payload["id"]
       if client_id_valid?(id)
-        logger.info("Received client_started message from id #{id}")
+        logger.info("Received client_started message from id '#{id}'")
       else
         msg = "Received client_started message from unknown id '#{id}'"
         logger.error(msg)

data/lib/beetle/subscriber.rb

@@ -1,3 +1,6 @@
+require 'amqp'
+require 'mq'
+
 module Beetle
   # Manages subscriptions and message processing on the receiver side of things.
   class Subscriber < Base
@@ -55,7 +58,7 @@ module Beetle
     end
 
     def queues_for_exchanges(exchanges)
-      @client.exchanges.slice(*exchanges).map{|_, opts| opts[:queues]}.flatten.uniq
+      @client.exchanges.slice(*exchanges).map{|_, opts| opts[:queues]}.flatten.compact.uniq
     end
 
     def create_exchanges(exchanges)
@@ -107,9 +110,9 @@ module Beetle
           message_options = opts.merge(:server => server, :store => @client.deduplication_store)
           m = Message.new(amqp_queue_name, header, data, message_options)
           result = m.process(processor)
-          if result.recover?
+          if result.reject?
             sleep 1
-            mq(server).recover
+            header.reject(:requeue => true)
           elsif reply_to = header.properties[:reply_to]
             # require 'ruby-debug'
             # Debugger.start
@@ -122,9 +125,6 @@ module Beetle
           Beetle::reraise_expectation_errors!
           # swallow all exceptions
           logger.error "Beetle: internal error during message processing: #{$!}: #{$!.backtrace.join("\n")}"
-        ensure
-          # support amqp metrics logging if we have it
-          logger.agent.flush if logger.respond_to?(:agent)
         end
       end
     end

data/lib/beetle.rb

@@ -1,6 +1,4 @@
 $:.unshift(File.expand_path('..', __FILE__))
-require 'amqp'
-require 'mq'
 require 'bunny-ext'
 require 'uuid4r'
 require 'active_support'

data/test/beetle/message_test.rb

@@ -543,7 +543,7 @@ module Beetle
       handler.expects(:process_failure).never
       result = message.process(handler)
       assert_equal RC::HandlerCrash, result
-      assert result.recover?
+      assert result.reject?
       assert !result.failure?
     end
 
@@ -561,7 +561,7 @@ module Beetle
       failback.expects(:call).once
       result = message.process(handler)
       assert_equal RC::AttemptsLimitReached, result
-      assert !result.recover?
+      assert !result.reject?
       assert result.failure?
     end
 
@@ -578,7 +578,7 @@ module Beetle
       failback.expects(:call).once
       result = message.process(handler)
       assert_equal RC::ExceptionsLimitReached, result
-      assert !result.recover?
+      assert !result.reject?
       assert result.failure?
     end
 

data/test/beetle/publisher_test.rb

@@ -19,7 +19,7 @@ module Beetle
       m = mock("dummy")
       expected_bunny_options = {
         :host => @pub.send(:current_host), :port => @pub.send(:current_port),
-        :logging => false, :user => "guest", :pass => "guest", :vhost => "/", :socket_timeout => 5.0
+        :logging => false, :user => "guest", :pass => "guest", :vhost => "/", :socket_timeout => 0
       }
       Bunny.expects(:new).with(expected_bunny_options).returns(m)
       m.expects(:start)
@@ -189,6 +189,27 @@ module Beetle
       @pub.expects(:publish_with_failover).with("mama-exchange", "mama", @data, @opts).returns(1)
       assert_equal 1, @pub.publish("mama", @data)
     end
+
+    test "failover publishing should raise an exception if the message was published to no server" do
+      failover = sequence("failover")
+      @pub.servers = ["someserver", "someotherserver"]
+      @pub.server = "someserver"
+
+      e = mock("exchange")
+      @pub.expects(:exchange).with("mama-exchange").returns(e).in_sequence(failover)
+      e.expects(:publish).raises(Bunny::ConnectionError).in_sequence(failover)
+      @pub.expects(:exchange).with("mama-exchange").returns(e).in_sequence(failover)
+      e.expects(:publish).raises(Bunny::ConnectionError).in_sequence(failover)
+      @pub.expects(:exchange).with("mama-exchange").returns(e).in_sequence(failover)
+      e.expects(:publish).raises(Bunny::ConnectionError).in_sequence(failover)
+      @pub.expects(:exchange).with("mama-exchange").returns(e).in_sequence(failover)
+      e.expects(:publish).raises(Bunny::ConnectionError).in_sequence(failover)
+
+      assert_raises Beetle::NoMessageSent do
+        @pub.publish_with_failover("mama-exchange", "mama", @data, @opts)
+      end
+    end
+
   end
 
   class PublisherQueueManagementTest < Test::Unit::TestCase

data/test/beetle/redis_configuration_server_test.rb

@@ -300,5 +300,14 @@ module Beetle
       @server.logger.expects(:error).with(msg)
       @server.client_started(payload)
     end
+
+    test "should log an info about received client_started client_started messages" do
+      payload = {"id" => "known-client"}
+      msg = "Received client_started message from id 'known-client'"
+      @server.logger.expects(:info).with(msg)
+      @server.expects(:client_id_valid?).with('known-client').returns(true)
+      @server.client_started(payload)
+    end
+
   end
 end

data/test/beetle/subscriber_test.rb

@@ -44,7 +44,12 @@ module Beetle
       assert_equal 42, mqs[@sub.server]
     end
 
-    test "stop! should stop the event loop" do
+    test "stop! should close all amqp connections and then stop the event loop" do
+      connection1 = mock('con1')
+      connection1.expects(:close).yields
+      connection2 = mock('con2')
+      connection2.expects(:close).yields
+      @sub.instance_variable_set "@amqp_connections", [["server1", connection1], ["server2",connection2]]
       EM.expects(:stop_event_loop)
       @sub.send(:stop!)
     end
@@ -103,6 +108,10 @@ module Beetle
       @sub.send(:bind_queues, %W(x y))
     end
 
+    test "should not try to bind a queue for an exchange which has no queue" do
+      @client.register_message(:without_queue)
+      assert_equal [], @sub.send(:queues_for_exchanges, ["without_queue"])
+    end
   end
 
   class SubscriberExchangeManagementTest < Test::Unit::TestCase
@@ -160,15 +169,13 @@ module Beetle
       assert_nothing_raised { @callback.call(header, 'foo') }
     end
 
-    test "should call recover on the server when processing the handler returns true on recover?" do
+    test "should call reject on the message header when processing the handler returns true on recover?" do
       header = header_with_params({})
       result = mock("result")
-      result.expects(:recover?).returns(true)
+      result.expects(:reject?).returns(true)
       Message.any_instance.expects(:process).returns(result)
       @sub.expects(:sleep).with(1)
-      mq = mock("MQ")
-      mq.expects(:recover)
-      @sub.expects(:mq).with(@sub.server).returns(mq)
+      header.expects(:reject).with(:requeue => true)
       @callback.call(header, 'foo')
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: beetle
 version: !ruby/object:Gem::Version 
-  hash: 119
+  hash: 105
   prerelease: false
   segments: 
   - 0
   - 2
   - 9
-  - 6
-  version: 0.2.9.6
+  - 9
+  version: 0.2.9.9
 platform: ruby
 authors: 
 - Stefan Kaes
@@ -19,7 +19,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-01 00:00:00 +01:00
+date: 2010-12-22 00:00:00 +01:00
 default_executable: beetle
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -62,12 +62,12 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 13
         segments: 
         - 0
         - 6
-        - 2
-        version: 0.6.2
+        - 5
+        version: 0.6.5
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency