right_amqp 0.3.1 → 0.3.2

data/lib/right_amqp/amqp/client.rb

@@ -6,6 +6,7 @@ module AMQP
 
   module BasicClient
     def process_frame frame
+      @last_data_received = Time.now
       if mq = channels[frame.channel]
         mq.process_frame(frame)
         return
@@ -48,8 +49,7 @@ module AMQP
 
       when Frame::Heartbeat
         logger.debug("[amqp] Received heartbeat from broker #{@settings[:identity]}")
-        @last_server_heartbeat = Time.now
-
+        @last_heartbeat_received = @last_data_received
       end
 
       # Make callback now that handshake with the broker has completed
@@ -111,21 +111,48 @@ module AMQP
     end
 
     def init_heartbeat
-      logger.debug("[amqp] Initializing heartbeat for broker #{@settings[:identity]} to #{@settings[:heartbeat]}")
-      @last_server_heartbeat = Time.now
-
-      @timer.cancel if @timer
-      @timer = EM::PeriodicTimer.new(@settings[:heartbeat]) do
-        if connected?
-          if @last_server_heartbeat < (Time.now - (@settings[:heartbeat] * 2))
-            log "Reconnecting due to missing server heartbeats"
-            logger.info("[amqp] Reconnecting to broker #{@settings[:identity]} due to missing server heartbeats")
-            reconnect(true)
-          else
-            logger.debug("[amqp] Sending heartbeat to broker #{@settings[:identity]}")
-            @last_server_heartbeat = Time.now
-            send AMQP::Frame::Heartbeat.new, :channel => 0
+      # Randomly offset start of heartbeat timer to help separate heartbeat
+      # activity when there are multiple broker connections active
+      EM.add_timer(rand(@settings[:heartbeat])) do
+        begin
+          # While connected, a heartbeat or some other data is expected to be received from
+          # the broker at least every 2 x :heartbeat seconds, otherwise the connection is
+          # assumed to be broken and therefore is closed to cause an automatic reconnect.
+          # While connected, this client will send a heartbeat every :heartbeat
+          # seconds regardless of any other send activity. The RabbitMQ broker will behave
+          # similarly and drop the connection if it does not receive a heartbeat or other
+          # data in time.
+          logger.info("[amqp] Initializing heartbeat for broker #{@settings[:identity]} to #{@settings[:heartbeat]} sec")
+
+          timeout_factor = 2
+          @heartbeat_timer.cancel if @heartbeat_timer
+          @heartbeat_timer = EM::PeriodicTimer.new(@settings[:heartbeat]) do
+            begin
+              if connected?
+                now = Time.now
+                if @last_data_received < (now - (@settings[:heartbeat] * timeout_factor))
+                  data_received = (now - @last_data_received).to_i if @last_data_received
+                  heartbeat_received = (now - @last_heartbeat_received).to_i if @last_heartbeat_received
+                  heartbeat_sent = (now - @last_heartbeat_sent).to_i if @last_heartbeat_sent
+                  logger.info("[amqp] Reconnecting to broker #{@settings[:identity]} due to heartbeat timeout: " +
+                              "last data received #{data_received.inspect} sec ago, " +
+                              "last heartbeat received #{heartbeat_received.inspect} sec ago, " +
+                              "last heartbeat sent #{heartbeat_sent.inspect} sec ago")
+                  close_connection # which results in an unbind and an automatic reconnect
+                else
+                  logger.debug("[amqp] Sending heartbeat to broker #{@settings[:identity]}")
+                  send AMQP::Frame::Heartbeat.new, :channel => 0
+                  @last_heartbeat_sent = Time.now
+                end
+              else
+                logger.debug("[amqp] Skipping heartbeat check for broker #{@settings[:identity]} because disconnected")
+              end
+            rescue Exception => e
+              logger.error("[amqp] Failed heartbeat check (#{e})\n" + e.backtrace.join("\n"))
+            end
           end
+        rescue Exception => e
+          logger.error("[amqp] Failed heartbeat initialization (#{e})\n" + e.backtrace.join("\n"))
         end
       end
     end
@@ -193,6 +220,8 @@ module AMQP
     #:startdoc:
 
     def close &on_disconnect
+      @heartbeat_timer.cancel if @heartbeat_timer
+      @heartbeat_timer = nil
       if on_disconnect
         @closing = true
         @on_disconnect = proc{
@@ -218,7 +247,13 @@ module AMQP
     def reconnect force = false
       if @reconnecting and not force
         # Wait after first reconnect attempt and in between each subsequent attempt
-        EM.add_timer(@settings[:reconnect_interval] || 5) { reconnect(true) }
+        EM.add_timer(@settings[:reconnect_interval] || 5) do
+          begin
+            reconnect(true)
+          rescue Exception => e
+            logger.exception("[amqp] Failed to reconnect", e, :trace)
+          end
+        end
         return
       end
 
@@ -236,7 +271,13 @@ module AMQP
         again = again.call if again.is_a?(Proc)
         if again.is_a?(Numeric)
           # Wait before making initial reconnect attempt
-          EM.add_timer(again) { reconnect(true) }
+          EM.add_timer(again) do
+            begin
+              reconnect(true)
+            rescue Exception => e
+              logger.exception("[amqp] Failed to reconnect", e, :trace)
+            end
+          end
           return
         elsif ![nil, true].include?(again)
           raise ::AMQP::Error, "Could not interpret :reconnect_delay => #{again.inspect}; expected nil, true, or Numeric"

data/right_amqp.gemspec

@@ -24,8 +24,8 @@ require 'rubygems'
 
 Gem::Specification.new do |spec|
   spec.name      = 'right_amqp'
-  spec.version   = '0.3.1'
-  spec.date      = '2012-06-08'
+  spec.version   = '0.3.2'
+  spec.date      = '2012-10-04'
   spec.authors   = ['Lee Kirchhoff']
   spec.email     = 'lee@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_amqp'

data/spec/amqp/client_extensions_spec.rb

@@ -0,0 +1,255 @@
+#
+# Copyright (c) 2009-2012 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'lib', 'right_amqp'))
+
+describe AMQP::Client do
+
+  include RightAMQP::SpecHelper
+  include FlexMock::ArgumentTypes
+
+  context 'reconnect' do
+
+    class SUT
+      include AMQP::Client
+
+      attr_accessor :reconnecting, :settings, :channels
+    end
+
+    before(:each) do
+      setup_logger
+      @sut = flexmock(SUT.new)
+      @sut.reconnecting = false
+      @sut.settings = {:host => 'testhost', :port=>'12345'}
+      @sut.channels = {}
+
+      @sut.should_receive(:initialize)
+    end
+
+    context 'with no :reconnect_delay' do
+      it 'should reconnect immediately' do
+        flexmock(EM).should_receive(:reconnect).once
+        flexmock(EM).should_receive(:add_timer).never
+
+        @sut.reconnect()
+      end
+    end
+
+    context 'with a :reconnect_delay of true' do
+      it 'should reconnect immediately' do
+        @sut.settings[:reconnect_delay] = true
+
+        flexmock(EM).should_receive(:reconnect).once
+        flexmock(EM).should_receive(:add_timer).never
+
+        @sut.reconnect()
+      end
+    end
+
+    context 'with a :reconnect_delay of 15 seconds' do
+      it 'should schedule a reconnect attempt in 15s' do
+        @sut.settings[:reconnect_delay] = 15
+
+        flexmock(EM).should_receive(:reconnect).never
+        flexmock(EM).should_receive(:add_timer).with(15, Proc).once
+
+        @sut.reconnect()
+      end
+    end
+
+    context 'with a :reconnect_delay containing a Proc that returns 30' do
+      it 'should schedule a reconnect attempt in 30s' do
+        @sut.settings[:reconnect_delay] = Proc.new {30}
+
+        flexmock(EM).should_receive(:reconnect).never
+        flexmock(EM).should_receive(:add_timer).with(30, Proc).once
+
+        @sut.reconnect()
+      end
+    end
+
+    context 'with a :reconnect_interval of 5 seconds'  do
+      it 'should schedule reconnect attempts on a 5s interval' do
+        @sut.reconnecting = true
+        @sut.settings[:reconnect_delay] = 15
+        @sut.settings[:reconnect_interval] = 5
+
+        flexmock(EM).should_receive(:reconnect).never
+        flexmock(EM).should_receive(:add_timer).with(5, Proc).once
+
+        @sut.reconnect()
+      end
+    end
+
+  end
+
+  context "heartbeat" do
+
+    class SUT
+      include AMQP::Client
+
+      attr_accessor :connected, :reconnecting, :settings, :channels, :heartbeat_timer
+      attr_accessor :last_data_received, :last_heartbeat_received, :last_heartbeat_sent
+    end
+
+    before(:each) do
+      setup_logger
+      @sut = flexmock(SUT.new)
+      @sut.reconnecting = false
+      @sut.channels = {}
+      @sut.should_receive(:initialize)
+      @sut.should_receive(:send_data)
+
+      @tune_payload = flexmock("tune payload", :class => AMQP::Protocol::Connection::Tune)
+      @method_frame = flexmock("method frame", :class => AMQP::Frame::Method, :channel => 0, :payload => @tune_payload)
+      @heartbeat_frame = flexmock("heartbeat frame", :class => AMQP::Frame::Heartbeat, :channel => 0)
+    end
+
+    context "with no :heartbeat setting" do
+      it "should never initialize heartbeat" do
+        @sut.should_receive(:init_heartbeat).never
+        @sut.connection_completed
+      end
+
+      it "should send :heartbeat setting of 0 to broker" do
+        flexmock(AMQP::Protocol::Connection::TuneOk).should_receive(:new).with(hsh(:heartbeat => 0))
+        @sut.process_frame(@method_frame)
+      end
+    end
+
+    context "with a :heartbeat setting of 0" do
+      before(:each) do
+        @sut.settings = {:heartbeat => 0}
+      end
+
+      it "should never initialize heartbeat" do
+        @sut.should_receive(:init_heartbeat).never
+        @sut.connection_completed
+      end
+
+      it "should send :heartbeat setting of 0 to broker" do
+        flexmock(AMQP::Protocol::Connection::TuneOk).should_receive(:new).with(hsh(:heartbeat => 0))
+        @sut.process_frame(@method_frame)
+      end
+    end
+
+    context "with a :heartbeat setting > 0" do
+
+      before(:each) do
+        @now = Time.at(1000000)
+        flexmock(Time).should_receive(:now).and_return(@now).by_default
+        @periodic_timer = flexmock("periodic timer")
+        flexmock(EM::PeriodicTimer).should_receive(:new).and_return(@periodic_timer).and_yield.by_default
+        flexmock(EM::PeriodicTimer).should_receive(:cancel).by_default
+        flexmock(EM).should_receive(:add_timer).and_yield.by_default
+
+        @heartbeat = 30
+        @sut.settings = {:heartbeat => @heartbeat, :identity => "test-heartbeat"}
+        @sut.connected = true
+        @sut.last_data_received = @now
+        @sut.last_heartbeat_received = nil
+        @sut.last_heartbeat_sent = nil
+      end
+
+      context "and opened connection is being tuned" do
+        it "should send :heartbeat setting to broker" do
+          flexmock(AMQP::Protocol::Connection::TuneOk).should_receive(:new).with(hsh(:heartbeat => @heartbeat))
+          @sut.process_frame(@method_frame)
+        end
+      end
+
+      context "and connection has completed" do
+        it "should initialize heartbeat" do
+          @sut.should_receive(:init_heartbeat).once
+          @sut.connection_completed
+        end
+
+        it "should initialize timer to start at a random :heartbeat time" do
+          flexmock(EM).should_receive(:add_timer).with(on { |arg| arg >= 0 && arg < @heartbeat }, Proc).once
+          @sut.connection_completed
+        end
+      end
+
+      context "and the heartbeat timer initialization fails" do
+        it "should log exception" do
+          @logger.should_receive(:info).with(/Initializing heartbeat for broker/).once
+          @logger.should_receive(:error).with(/Failed heartbeat initialization/).once
+          flexmock(EM::PeriodicTimer).should_receive(:new).and_raise(Exception)
+          @sut.init_heartbeat
+        end
+      end
+
+      context "and the timer fires when not connected" do
+        it "should log a debug message but not perform check" do
+          @logger.should_receive(:debug).with(/Skipping heartbeat check for broker/).once
+          flexmock(EM::PeriodicTimer).should_receive(:new).and_yield
+          @sut.connected = false
+          @sut.init_heartbeat
+        end
+      end
+
+      context "and the timer fires when connected" do
+
+        context "and no data has been received in 2 x :heartbeat interval" do
+          it "should close connection to force reconnect" do
+            @sut.last_data_received = @now - (@heartbeat * 2) - 1
+            @logger.should_receive(:info).with(/Initializing heartbeat for broker/).once
+            @logger.should_receive(:info).with(/Reconnecting to broker/).once
+            @sut.should_receive(:close_connection).once
+            @sut.init_heartbeat
+          end
+        end
+
+        context "and timeout check passes" do
+          it "should send heartbeat" do
+            @logger.should_receive(:debug).with(/Sending heartbeat to broker/).once
+            @sut.should_receive(:send).with(AMQP::Frame::Heartbeat, :channel => 0).once
+            @sut.init_heartbeat
+          end
+        end
+
+        context "and the heartbeat check fails" do
+          it "should log exception" do
+            @logger.should_receive(:error).with(/Failed heartbeat check/).once
+            @sut.should_receive(:send).and_raise(Exception)
+            @sut.init_heartbeat
+          end
+        end
+
+      end
+
+      context "and connection is being closed" do
+        it "should cancel heartbeat timer" do
+          @periodic_timer.should_receive(:cancel).once
+          @sut.connection_completed
+          @sut.heartbeat_timer.should_not be_nil
+          @sut.close
+          @sut.heartbeat_timer.should be_nil
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -44,6 +44,7 @@ module RightAMQP
     def setup_logger
       @logger = flexmock("logger")
       @logger.should_receive(:level).and_return(:info).by_default
+      @logger.should_receive(:exception).by_default.and_return { |m| raise m }
       @logger.should_receive(:error).by_default.and_return { |m| raise m }
       @logger.should_receive(:warning).by_default.and_return { |m| raise m }
       @logger.should_receive(:info).by_default
@@ -53,4 +54,4 @@ module RightAMQP
 
   end
 
-end
+end

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: right_amqp
 version: !ruby/object:Gem::Version 
-  hash: 17
-  prerelease: false
-  segments: 
-  - 0
-  - 3
-  - 1
-  version: 0.3.1
+  prerelease: 
+  version: 0.3.2
 platform: ruby
 authors: 
 - Lee Kirchhoff
@@ -15,47 +10,33 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-08 00:00:00 -07:00
-default_executable: 
+date: 2012-10-04 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
+  name: right_support
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 1
-        - 2
         version: "1.2"
     - - <
       - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 3
-        - 0
         version: "3.0"
-  requirement: *id001
-  name: right_support
-  prerelease: false
   type: :runtime
+  prerelease: false
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
+  name: eventmachine
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 59
-        segments: 
-        - 0
-        - 12
-        - 10
         version: 0.12.10
-  requirement: *id002
-  name: eventmachine
-  prerelease: false
   type: :runtime
+  prerelease: false
+  version_requirements: *id002
 description: |
   RightAMQP provides a high availability client for interfacing with the
   RightScale RabbitMQ broker using the AMQP protocol. The AMQP version on which
@@ -100,12 +81,11 @@ files:
 - lib/right_amqp/mq/queue.rb
 - lib/right_amqp/mq/rpc.rb
 - right_amqp.gemspec
-- spec/amqp/client_reconnect_spec.rb
+- spec/amqp/client_extensions_spec.rb
 - spec/ha_client/broker_client_spec.rb
 - spec/ha_client/ha_broker_client_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-has_rdoc: true
 homepage: https://github.com/rightscale/right_amqp
 licenses: []
 
@@ -122,25 +102,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 57
-      segments: 
-      - 1
-      - 8
-      - 7
       version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
+      hash: 60400910479135744
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: Client for interfacing to RightScale RabbitMQ broker using AMQP

data/spec/amqp/client_reconnect_spec.rb

@@ -1,105 +0,0 @@
-#
-# Copyright (c) 2009-2012 RightScale Inc
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'lib', 'right_amqp'))
-
-describe AMQP::Client do
-
-  context 'with an incorrect AMQP password' do
-
-    include RightAMQP::SpecHelper
-
-    class SUT
-      include AMQP::Client
-
-      attr_accessor :reconnecting, :settings, :channels
-    end
-
-    before(:each) do
-      setup_logger
-      @sut = flexmock(SUT.new)
-      @sut.reconnecting = false
-      @sut.settings = {:host => 'testhost', :port=>'12345'}
-      @sut.channels = {}
-
-      @sut.should_receive(:initialize)
-    end
-
-    context 'and no :reconnect_delay' do
-      it 'should reconnect immediately' do
-        flexmock(EM).should_receive(:reconnect).once
-        flexmock(EM).should_receive(:add_timer).never
-
-        @sut.reconnect()
-      end
-    end
-
-    context 'and a :reconnect_delay of true' do
-      it 'should reconnect immediately' do
-        @sut.settings[:reconnect_delay] = true
-
-        flexmock(EM).should_receive(:reconnect).once
-        flexmock(EM).should_receive(:add_timer).never
-
-        @sut.reconnect()
-      end
-    end
-
-    context 'and a :reconnect_delay of 15 seconds' do
-      it 'should schedule a reconnect attempt in 15s' do
-        @sut.settings[:reconnect_delay] = 15
-
-        flexmock(EM).should_receive(:reconnect).never
-        flexmock(EM).should_receive(:add_timer).with(15, Proc).once
-
-        @sut.reconnect()
-      end
-    end
-
-    context 'and a :reconnect_delay containing a Proc that returns 30' do
-      it 'should schedule a reconnect attempt in 30s' do
-        @sut.settings[:reconnect_delay] = Proc.new {30}
-
-        flexmock(EM).should_receive(:reconnect).never
-        flexmock(EM).should_receive(:add_timer).with(30, Proc).once
-
-        @sut.reconnect()
-      end
-    end
-
-    context 'and a :reconnect_interval of 5 seconds'  do
-      it 'should schedule reconnect attempts on a 5s interval' do
-        @sut.reconnecting = true
-        @sut.settings[:reconnect_delay] = 15
-        @sut.settings[:reconnect_interval] = 5
-
-        flexmock(EM).should_receive(:reconnect).never
-        flexmock(EM).should_receive(:add_timer).with(5, Proc).once
-
-        @sut.reconnect()
-      end
-    end
-
-  end
-
-end