amqp-failover 0.0.1

data/lib/amqp/failover_client.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+
+module AMQP
+  module FailoverClient
+    include AMQP::BasicClient
+    
+    attr_accessor :failover
+    attr_reader :fallback_monitor
+    
+    attr_accessor :settings
+    attr_accessor :on_disconnect
+    
+    def self.extended(base)
+      if (base.failover = base.settings.delete(:failover))
+        base.on_disconnect = base.method(:disconnected)
+      end
+    end
+    
+    def failover_switch
+      if (new_settings = @failover.from(@settings))
+        log_message = "Could not connect to or lost connection to server #{@settings[:host]}:#{@settings[:port]}. " +
+                      "Attempting connection to: #{new_settings[:host]}:#{new_settings[:port]}"
+        logger.error(log_message)
+        logger.info(log_message)
+        
+        if @failover.options[:fallback] && @failover.primary == @settings
+          fallback(@failover.primary, @failover.fallback_interval)
+        end
+        @settings = new_settings
+        reconnect
+      else
+        raise Error, "Could not connect to server #{@settings[:host]}:#{@settings[:port]}"
+      end
+    end
+    
+    def logger
+      Failover.logger
+    end
+    
+    def configs
+      @failover.configs if @failover
+    end
+    
+    def clean_exit(msg = nil)
+      msg ||= "clean exit"
+      logger.info(msg)
+      logger.error(msg)
+      Process.exit
+    end
+    
+    def fallback(conf = {}, retry_interval = nil)
+      @fallback_monitor = Failover::ServerDiscovery.monitor(conf, retry_interval) do
+        fallback_callback.call(conf, retry_interval)
+      end
+    end
+    
+    def fallback_callback
+      #TODO: Figure out a way to artificially trigger EM to disconnect on fallback without channels being closed.
+      @fallback_callback ||= proc { |conf, retry_interval|
+        clean_exit("Primary server (#{conf[:host]}:#{conf[:port]}) is back. " +
+                   "Performing clean exit to be relaunched with primary config.")
+      }
+    end
+    attr_writer :fallback_callback
+    
+    #TODO: Figure out why I originally needed this
+    # def process_frame(frame)
+    #   if mq = channels[frame.channel]
+    #     mq.process_frame(frame)
+    #     return
+    #   end
+    #   
+    #   if frame.is_a?(AMQP::Frame::Method) && (method = frame.payload).is_a?(AMQP::Protocol::Connection::Close)
+    #     if method.reply_text =~ /^NOT_ALLOWED/
+    #       raise AMQP::Error, "#{method.reply_text} in #{::AMQP::Protocol.classes[method.class_id].methods[method.method_id]}"
+    #     end
+    #   end
+    #   super(frame)
+    # end
+    
+  end # FailoverClient
+end # AMQP
+

data/spec/integration/basic_spec.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+require 'spec_helper'
+require 'mq'
+require 'amqp'
+require 'amqp/server'
+require 'server_helper'
+
+describe "Basic AMQP connection with FailoverClient loaded" do
+  
+  after(:each) do
+    ServerHelper.clear_logs
+  end
+  
+  it "should be using FailoverClient" do
+    AMQP.client.should == AMQP::FailoverClient
+  end
+  
+  it "should be able to connect" do
+    EM.run {
+      port = 15672
+      timeout = 2
+      serv = start_server(port)
+      EM.add_timer(1.5) {
+        conn = AMQP.connect(:host => 'localhost', :port => 15672)
+        EM.add_timer(0.1) {
+          conn.should be_connected
+          serv.stop
+          log = serv.log
+          log.size.should == 3
+          (0..2).each { |i| log[i]['method'].should == "send" }
+          log[0]['class'].should == 'AMQP::Protocol::Connection::Start'
+          log[1]['class'].should == 'AMQP::Protocol::Connection::Tune'
+          log[2]['class'].should == 'AMQP::Protocol::Connection::OpenOk'
+          EM.stop
+        }
+      }
+    }
+  end
+  
+  it "should be able to connect and get disconnected" do
+    EM.run {
+      serv = start_server(25672)
+      EM.add_timer(0.1) {
+        conn = AMQP.connect(:host => 'localhost', :port => 25672)
+        EM.add_timer(0.1) {
+          conn.should be_connected
+          serv.stop
+          EM.add_timer(0.1) {
+            conn.should_not be_connected
+            EM.stop
+          }
+        }
+      }
+    }
+  end
+  
+end

data/spec/integration/failover_spec.rb

@@ -0,0 +1,141 @@
+# encoding: utf-8
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+require 'spec_helper'
+require 'amqp/server'
+require 'server_helper'
+require 'logger_helper'
+
+describe "Failover support loaded into AMQP gem" do
+  
+  before(:each) do
+    @flog = LoggerHelper.new
+    AMQP::Failover.logger = @flog
+  end
+  
+  after(:each) do
+    ServerHelper.clear_logs
+    AMQP::Failover.logger = nil
+  end
+  
+  it "should be able to connect" do
+    port1 = 15672
+    EM.run {
+      serv = start_server(port1)
+      EM.add_timer(0.1) {
+        conn = AMQP.connect(:host => 'localhost', :port => port1)
+        conn.failover.should be_nil
+        EM.add_timer(0.1) {
+          conn.should be_connected
+          EM.stop
+        }
+      }
+    }
+  end
+  
+  it "should be able to connect and failover" do
+    port1 = 25672
+    port2 = 35672
+    EM.run {
+      # start mock amqp servers
+      serv1 = start_server(port1)
+      serv2 = start_server(port2)
+      EM.add_timer(0.1) {
+        # start amqp client connection and make sure it's picked the right config
+        conn = AMQP.connect({:hosts => [{:port => port1}, {:port => port2}]})
+        conn.failover.primary[:port].should == port1
+        conn.settings[:port].should == port1
+        conn.settings.should == conn.failover.primary
+        EM.add_timer(0.1) {
+          # make sure client connected to the correct server, then kill server
+          conn.should be_connected
+          serv1.log.should have(3).items
+          serv2.log.should have(0).items
+          serv1.stop
+          EM.add_timer(0.1) {
+            # make sure client performed a failover when primary server died
+            conn.should be_connected
+            [:error, :info].each do |i|
+              @flog.send("#{i}_log").should have(1).item
+              @flog.send("#{i}_log")[0][0].should match(/connect to or lost connection.+#{port1}.+attempting connection.+#{port2}/i)
+            end
+            conn.settings[:port].should == port2
+            serv1.log.should have(3).items
+            serv2.log.should have(3).items
+            conn.close
+            EM.add_timer(0.1) {
+              serv2.stop
+              EM.stop
+            }
+          }
+        }
+      }
+    }
+  end
+  
+  it "should be able to fallback when primary server returns" do
+    port1 = 45672
+    port2 = 55672
+    lambda {
+      EM.run {
+        # start mock amqp servers
+        serv1 = start_server(port1)
+        serv2 = start_server(port2)
+        EM.add_timer(0.1) {
+          # start amqp client connection and make sure it's picked the right config
+          conn = AMQP.connect({:hosts => [{:port => port1}, {:port => port2}], :fallback => true, :fallback_interval => 0.1})
+          conn.failover.primary[:port].should == port1
+          conn.settings[:port].should == port1
+          conn.settings.should == conn.failover.primary
+          EM.add_timer(0.1) {
+            # make sure client connected to the correct server, then kill server
+            conn.should be_connected
+            serv1.log.should have(3).items
+            serv2.log.should have(0).items
+            serv1.stop
+            EM.add_timer(0.1) {
+              # make sure client performed a failover when primary server died
+              conn.should be_connected
+              [:error, :info].each do |i|
+                @flog.send("#{i}_log").should have(1).item
+                @flog.send("#{i}_log")[0][0].should match(/connect to or lost connection.+#{port1}.+attempting connection.+#{port2}/i)
+              end
+              conn.settings[:port].should == port2
+              serv1.log.should have(3).items
+              serv2.log.should have(3).items
+              serv3 = start_server(port1)
+              EM.add_timer(0.2) {
+                # by this point client should have raised a SystemExit exception
+                serv2.stop
+                EM.stop
+              }
+            }
+          }
+        }
+      }
+    }.should raise_error(SystemExit, "exit")
+    [:error, :info].each do |i|
+      @flog.send("#{i}_log").should have(2).item
+      @flog.send("#{i}_log")[1][0].should match(/primary server.+45672.+performing clean exit/i)
+    end
+  end
+  
+  it "should abide to :primary_config option" do
+    port1 = 75672
+    port2 = 65672
+    EM.run {
+      serv = start_server(port1)
+      EM.add_timer(0.1) {
+        conn = AMQP.connect({:hosts => [{:port => port1}, {:port => port2}], :primary_config => 1})
+        conn.failover.primary[:port].should == port2
+        conn.settings[:port].should == port2
+        conn.settings.should == conn.failover.primary
+        EM.add_timer(0.1) {
+          conn.should be_connected
+          EM.stop
+        }
+      }
+    }
+  end
+  
+end

data/spec/logger_helper.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+class LoggerHelper
+  
+  attr_accessor :error_log
+  attr_accessor :info_log
+  
+  def info(*args)
+    @info_log ||= []
+    @info_log << args
+  end
+  
+  def error(*args)
+    @error_log ||= []
+    @error_log << args
+  end
+  
+end

data/spec/server_helper.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'mq'
+require 'amqp'
+require 'amqp/server'
+require 'json'
+
+class ServerHelper
+  
+  attr_accessor :stdin
+  attr_accessor :stdout
+  attr_accessor :stderr
+  attr_accessor :pid
+  
+  
+  def initialize(port = nil, timeout = nil)
+    @port = port
+    @timout = timeout
+    File.open(log_file, 'w') {}
+    @pid = start(port, timeout)
+  end
+  
+  def self.clear_logs
+    Dir.glob(File.expand_path('server_helper*.log', File.dirname(__FILE__))).each do |file|
+      File.delete(file)
+    end
+  end
+  
+  def start(port = nil, timeout = nil)
+    port ||= 15672
+    timeout ||= 2
+    EM.fork_reactor {
+      $PORT = port
+      EM.start_server('localhost', port, AmqpServer)
+      EM.add_timer(timeout) { EM.stop }
+    }
+  end
+  
+  def stop
+    Process.kill('TERM', @pid)
+  end
+  
+  def kill
+    Process.kill('KILL', @pid)
+  end
+  
+  def log
+    File.open(log_file).to_a.map{ |l| JSON.parse(l) }
+  end
+  
+  def log_file
+    File.expand_path("server_helper-port#{@port}.log", File.dirname(__FILE__))
+  end
+  
+end
+
+module AmqpServer
+  include AMQP::Server
+
+  # customize log output
+  def log(*args)
+    args = {:method => args[0], :class => args[1].payload.class, :pid => Process.pid}
+    filename = File.expand_path("server_helper-port#{$PORT}.log", File.dirname(__FILE__))
+    File.open(filename, 'a') do |f|
+      f.write("#{args.to_json}\n")
+    end
+  end
+end
+
+#
+# Helper methods
+#
+
+def start_server(port = nil, timeout = nil)
+  ServerHelper.new(port, timeout)
+end

data/spec/spec_helper.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+# add project-relative load paths
+$LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+# require stuff
+require 'rubygems'
+
+begin
+  require 'mq'
+rescue LoadError => e
+  require 'amqp'
+end
+require 'amqp/failover'
+
+require 'rspec'
+require 'rspec/autorun'
+
+
+#
+# Helper methods
+#
+
+def wait_while(timeout = 10, retry_interval = 0.1, &block)
+  start = Time.now
+  while block.call
+    break if (Time.now - start).to_i >= timeout
+    sleep(retry_interval)
+  end
+end
+
+# stolen from Pid::running? from daemons gem
+def pid_running?(pid)
+  return false unless pid
+  
+  # Check if process is in existence
+  # The simplest way to do this is to send signal '0'
+  # (which is a single system call) that doesn't actually
+  # send a signal
+  begin
+    Process.kill(0, pid)
+    return true
+  rescue Errno::ESRCH
+    return false
+  rescue ::Exception   # for example on EPERM (process exists but does not belong to us)
+    return true
+  end
+end

data/spec/unit/amqp/failover/config_spec.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+require 'spec_helper'
+
+describe 'AMQP::Failover::Config' do
+  
+  before(:each) do
+    configs = [
+      {:host => 'rabbit0.local'},
+      {:host => 'rabbit1.local'},
+      {:host => 'rabbit2.local', :port => 5673}
+    ]
+    @configs = configs.map { |conf| AMQP.settings.merge(conf) }
+    @fail = AMQP::Failover.new(@configs)
+  end
+  
+  it "should initialize" do
+    fail = AMQP::Failover::Config.new(@configs[0])
+    fail.should == @configs[0]
+    fail.last_fail.should be_nil
+    
+    now = Time.now
+    fail = AMQP::Failover::Config.new(@configs[1], now)
+    fail.should == @configs[1]
+    fail.last_fail.should == now
+  end
+  
+  it "should order properly with #<=>" do
+    one_hour_ago = (Time.now - 3600)
+    two_hours_ago = (Time.now - 7200)
+
+    fail = [ AMQP::Failover::Config.new(@configs[0]),
+             AMQP::Failover::Config.new(@configs[1], one_hour_ago),
+             AMQP::Failover::Config.new(@configs[2], two_hours_ago) ]
+    
+    (fail[1] <=> fail[0]).should == -1
+    (fail[0] <=> fail[0]).should == 0
+    (fail[0] <=> fail[1]).should == 1
+    
+    (fail[1] <=> fail[2]).should == -1
+    (fail[1] <=> fail[1]).should == 0
+    (fail[2] <=> fail[1]).should == 1
+    
+    fail.sort[0].last_fail.should == one_hour_ago
+    fail.sort[1].last_fail.should == two_hours_ago
+    fail.sort[2].last_fail.should == nil
+  end
+  
+  it "should be ordered by last_fail" do
+    result = [ AMQP::Failover::Config.new(@configs[1], (Time.now - 60)),
+               AMQP::Failover::Config.new(@configs[2], (Time.now - (60*25))),
+               AMQP::Failover::Config.new(@configs[0], (Time.now - 3600)) ]
+               
+    origin = [ AMQP::Failover::Config.new(@configs[0], (Time.now - 3600)),
+               AMQP::Failover::Config.new(@configs[1], (Time.now - 60)),
+               AMQP::Failover::Config.new(@configs[2], (Time.now - (60*25))) ]
+    origin.sort.should == result
+    
+    origin = [ AMQP::Failover::Config.new(@configs[0]),
+               AMQP::Failover::Config.new(@configs[1], (Time.now - 60)),
+               AMQP::Failover::Config.new(@configs[2], (Time.now - (60*25))) ]
+    origin.sort.should == result
+  end
+  
+end
+

data/spec/unit/amqp/failover/configurations_spec.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+require 'spec_helper'
+
+describe 'AMQP::Failover::Configurations' do
+  
+  before(:each) do
+    @conf = AMQP::Failover::Configurations.new
+    @raw_configs = [
+      {:host => 'rabbit0.local'},
+      {:host => 'rabbit1.local'},
+      {:host => 'rabbit2.local', :port => 5673}
+    ]
+    @configs = @raw_configs.map { |conf| AMQP.settings.merge(conf) }
+  end
+  
+  it "should initialize" do
+    confs = AMQP::Failover::Configurations.new(@raw_configs)
+    confs.each_with_index do |conf, i|
+      conf.should be_a(AMQP::Failover::Config)
+      conf.should == @configs[i]
+    end
+  end
+  
+  it "should set and get configs" do
+    @conf.primary_ref.should == 0
+    @conf.should have(0).items
+    
+    @conf.set(@raw_configs[0])
+    @conf.should have(1).items
+    @conf.get(0).should == @configs[0]
+    @conf[0].should == @configs[0]
+    
+    @conf.set(@raw_configs[1])
+    @conf.should have(2).items
+    @conf.get(1).should == @configs[1]
+    @conf[1].should == @configs[1]
+    
+    # should just create a ref, as config exists
+    @conf.set(@raw_configs[1], :the_one)
+    @conf.should have(2).items
+    @conf.get(1).should == @configs[1]
+    @conf[:the_one].should == @configs[1]
+    
+    @conf.load_array(@raw_configs)
+    @conf.should have(3).items
+    @conf.primary.should == @configs[0]
+    @conf.primary_ref = 1
+    @conf.primary.should == @configs[1]
+    @conf[:primary].should == @configs[1]
+  end
+  
+  it "should #find_next" do
+    @conf.load(@raw_configs)
+    @conf.should have(3).items
+    @conf.find_next(@configs[0]).should == @configs[1]
+    @conf.find_next(@configs[1]).should == @configs[2]
+    @conf.find_next(@configs[2]).should == @configs[0]
+  end
+  
+  it "should #load_hash" do
+    @conf.should have(0).items
+    @conf.load_hash(@raw_configs[0])
+    @conf.should have(1).items
+    @conf.primary.should == @configs[0]
+  end
+  
+  it "should #load_array" do
+    @conf.load_hash(:host => 'rabbid-rabbit')
+    @conf.should have(1).items
+    @conf.load_array(@raw_configs)
+    @conf.should have(3).items
+    @conf.should == @configs
+    @conf.primary.should == @configs[0]
+  end
+  
+end
+

data/spec/unit/amqp/failover/server_discovery_helper.rb

@@ -0,0 +1,31 @@
+class ServerDiscoveryHelper < AMQP::Failover::ServerDiscovery
+  
+  class << self
+    alias :real_start_monitoring :start_monitoring
+    def start_monitoring(*args, &block)
+      $called << :start_monitoring
+      real_start_monitoring(*args, &block)
+    end
+  end
+  
+  alias :real_initialize :initialize
+  def initialize(*args)
+    $called << :initialize
+    EM.start_server('127.0.0.1', 9999) if $start_count == 2
+    $start_count += 1
+    real_initialize(*args)
+  end
+  
+  alias :real_connection_completed :connection_completed
+  def connection_completed
+    $called << :connection_completed
+    real_connection_completed
+  end
+  
+  alias :real_close_connection :close_connection
+  def close_connection
+    $called << :close_connection
+    real_close_connection
+  end
+  
+end

data/spec/unit/amqp/failover/server_discovery_spec.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+require 'spec_helper'
+require 'server_discovery_helper'
+
+describe 'AMQP::Failover::ServerDiscovery' do
+  
+  before(:each) do
+    $called = []
+    $start_count = 0
+    @args = { :host => 'localhost', :port => 9999 }
+    @retry_interval = 0.01
+  end
+  
+  after(:all) do
+    $called = nil
+    $start_count = nil
+  end
+  
+  it "should initialize" do
+    EM.run {
+      EM.start_server('127.0.0.1', 9999)
+      @mon = ServerDiscoveryHelper.monitor(@args, @retry_interval) do
+        $called << :done_block
+        EM.stop_event_loop
+      end
+    }
+    $start_count.should == 1
+    $called.should have(5).items
+    $called.uniq.should have(5).items
+    $called.should include(:start_monitoring)
+    $called.should include(:initialize)
+    $called.should include(:connection_completed)
+    $called.should include(:close_connection)
+    $called.should include(:done_block)
+  end
+  
+  it "should retry on error" do
+    EM.run {
+      @mon = ServerDiscoveryHelper.monitor(@args, @retry_interval) do
+        $called << :done_block
+        EM.stop_event_loop
+      end
+    }
+    $start_count.should >= 3
+    $called.should have($start_count + 4).items
+    $called.uniq.should have(5).items
+    $called.should include(:start_monitoring)
+    $called.should include(:initialize)
+    $called.should include(:connection_completed)
+    $called.should include(:close_connection)
+    $called.should include(:done_block)
+  end
+  
+end