who_can 0.3.4

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+*.log
+.idea
+rabbitmq

data/.rspec

@@ -0,0 +1,4 @@
+--color
+--require ./spec/support/logging_progress_bar_formatter.rb
+--format Motionbox::LoggingProgressBarFormatter
+

data/.rvmrc

@@ -0,0 +1 @@
+rvm 1.9.2-p290@whocan --create

data/Gemfile

@@ -0,0 +1,8 @@
+source "http://rubygems.org"
+source "http://localhost:50000"
+
+# Specify your gem's dependencies in whocan.gemspec
+gemspec
+
+
+# vim:ft=ruby

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/lib/who_can.rb

@@ -0,0 +1,64 @@
+require 'rubygems'
+require 'amqp'
+require 'state_machine'
+require 'uuidtools'
+require 'thread'
+require 'monitor'
+require 'logger'
+require 'set'
+require 'deferred'
+
+%w[ core_ext/array/extract_options 
+    core_ext/module/aliasing        
+    core_ext/hash/reverse_merge   
+    core_ext/hash/indifferent_access ].each do |p|
+
+  require "active_support/#{p}"
+end
+
+module WhoCan
+  DEFAULT_PING_EXCHANGE = 'who_can.default.fanout'
+  DEFAULT_CONNECT_URL = 'amqp://127.0.0.1'
+
+  def self.connect_url
+    @config ||= DEFAULT_CONNECT_URL
+  end
+
+  def self.connect_url=(config)
+    @config = config
+  end
+
+  def self.new(*a, &b)
+    WhoCan::Base.new(*a, &b)
+  end
+
+  def self.logger
+    @logger ||= Logger.new('/dev/null').tap { |l| l.level = Logger::FATAL }
+  end
+
+  def self.logger=(log)
+    @logger = log
+  end
+
+  class WhoCanError < StandardError; end
+  class DelayMustBeSetError < WhoCanError
+    def initialize
+      super("delay must be set for on_ping Response object")
+    end
+  end
+
+  class ChannelPoolError < WhoCanError; end
+  class PoolIsNotOpenException < ChannelPoolError; end
+  class TimeoutError < WhoCanError; end
+end
+
+
+require 'who_can/logging'
+require 'who_can/base'
+require 'who_can/pinger'
+require 'who_can/responder'
+require 'who_can/heartbeater'
+require 'who_can/connection_wrapper'
+require 'who_can/connection_manager'
+
+

data/lib/who_can/base.rb

@@ -0,0 +1,72 @@
+module WhoCan
+  class Base
+    include Deferred::Accessors
+    include Logging
+
+    deferred_event :disconnection
+    attr_accessor :connection_opts
+
+    def initialize(connection_opts=nil)
+      @connection_opts = connection_opts
+      @connection = nil
+      if @connection_opts.is_a?(AMQP::Session)
+        @connection = @connection_opts
+      end
+#       logger.debug { "connection_opts: #{@connection_opts.inspect}" }
+      @responders = {}
+
+    end
+
+    def connection(&blk)
+      @connection ||= AMQP.connect(connection_opts || WhoCan.connect_url)
+      on_open(&blk) if blk
+      @connection
+    end
+    alias :connect! :connection
+
+    def on_open(&block)
+      connection.on_open(&block)
+    end
+
+    def start_monitoring!(opts = {})
+      @ekg = Heartbeater::EKG.new(connection)
+      @ekg.on_heartbeat_failure do
+        on_disconnection.succeed
+        close!
+      end
+      @ekg.start!
+    end
+
+    def new_pinger
+      WhoCan::Pinger.new(connection)
+    end
+
+    def new_responder(exchange_name)
+      WhoCan::Responder.new(connection, exchange_name)
+    end
+
+    def close!(&block)
+      unless @connection
+        block.call
+        return
+      end
+      cnx, @connection = @connection, nil
+      @channel = nil
+      on_disconnection(&block)
+      cnx.on_disconnection do
+        on_disconnection.succeed
+      end
+      cnx.close
+      on_disconnection
+    end
+
+    def clear_responder(ping_exchange_name)
+      if resp = @responders.delete(ping_exchange_name)      
+        resp.close!
+        true
+      end
+      false
+    end
+  end
+end
+

data/lib/who_can/connection_manager.rb

@@ -0,0 +1,268 @@
+module WhoCan
+  # This class handles setting up the AMQP connection. You must start the
+  # EventMachine reactor before running anything in this class
+  #
+  # Keeps two connections open, a primary and secondary, and reacts to changes
+  # in their state. In the non-failure case, when a client calls #connection on
+  # us, they will receive the primary connection (once it's been established).
+  # If the primary is down, or the connection can't be established for some
+  # reason, then the secondary is returned. 
+  #
+  # When a failover situation happens (i.e. the primary goes down) the
+  # on_failure callback will be fired. When that callback is fired, the 
+  # users of this class are expected to clean up and wait until the 
+  # on_recovery deferred is fired. The on_recovery deferred is fired when there
+  # is at least one open connection to a broker available. 
+  #
+  # @note this class *will* automatically fail-back when the primary is
+  #   re-established.
+  #
+  # TODO: determine how long to wait for connection to primary before failing
+  #       over to secondary.
+  #
+  class ConnectionManager
+    include Deferred::Accessors
+    include Logging
+    
+    DEFAULT_OPTIONS = {
+      :primary_grace_time => 5.0
+    }
+
+    deferred_event :open, :failure, :close
+
+    attr_reader :primary_opts, :secondary_opts, :primary, :secondary
+
+    attr_accessor :primary_grace_time
+
+    state_machine :state, :initial => :new do
+
+      event :start do
+        transition :new => :start_up
+      end
+
+      # when the secondary connects, it only matters to us if the primary
+      # hasn't been established yet or if it's in some transitional state
+      # 
+      event :secondary_connected do
+        transition :start_up  => :awaiting_primary, :unless => :primary_connected?
+        transition :degraded  => :connected
+        transition :down      => :failed_over
+      end
+
+      # fired when we're awaiting the primary to connect and it took too long
+      event :primary_wait_timeout do
+        transition :awaiting_primary => :failed_over
+      end
+
+      # when the primary connects we move into "connected" state and the
+      # clients can now start using the connection
+      #
+      event :primary_connected do
+        transition [:start_up, :down] => :degraded, :unless => :secondary_connected?
+        transition [:awaiting_primary, :failed_over] => :connected
+      end
+
+      event :secondary_failed do
+        # if we're in failed_over mode, transition to down
+        # if we're in connected, just ignore event
+        transition :connected => :degraded
+        transition :failed_over => :down
+      end
+
+      event :primary_failed do
+        transition :connected => :failed_over
+        transition :degraded =>  :down
+      end
+
+      event :close do
+        transition (all - [:new, :shutting_down, :closed]) => :shutting_down
+        transition :new => :closed
+      end
+
+      event :closed do
+        transition :shutting_down => :closed
+      end
+
+      # start trying to connect to the cluster
+      after_transition :new => :start_up, :do => [:do_connect_primary, :do_connect_secondary]
+
+      # the secondary has connected, start the timer to wait for the primary to connect
+      after_transition :start_up => :awaiting_primary, :do => :start_primary_timer
+
+      # the primary has not connected within the grace period, allow clients to connect
+      after_transition :awaiting_primary => :failed_over, :do => :enter_failed_over_state
+
+      # the primary has connected within the grace period, enter the connected state
+      after_transition :awaiting_primary => :connected,   :do => :become_usable!
+
+      #make sure we are up as soon as possible - don't wait for the secondary
+      after_transition :start_up => :degraded, :do => :become_usable!
+
+      #primary has returned
+      after_transition :failed_over => :connected, :do => [:fire_failure_deferred, :become_usable!]
+
+      # the primary fails, secondary is available
+      after_transition :connected => :failed_over, :do => [:fire_failure_deferred, :enter_failed_over_state, :do_connect_primary]
+
+      #the two states we can be in when the secondary is going down
+      after_transition :failed_over => :down, :do => [:fire_failure_deferred, :do_connect_secondary]
+      after_transition :connected => :degraded, :do => :do_connect_secondary
+
+      #primary went down, and secondary was already down
+      after_transition :degraded => :down, :do => [:fire_failure_deferred, :do_connect_primary, :do_connect_secondary]
+
+      #we've been down, now we're coming back up
+      after_transition :down => :failed_over, :do => :enter_failed_over_state
+      after_transition :down => :degraded, :do => :become_usable!
+
+      # shutdown states
+      after_transition [:awaiting_primary, :degraded, :failed_over, :down, :connected] => :shutting_down, :do => :do_close
+
+      # we are really closed
+      after_transition :shutting_down => :closed, :do => :fire_closed_callbacks
+
+      # logging of state transitions
+      around_transition do |cw,block|
+        orig_state = cw.state
+        block.call
+        logger.debug { "CONNECTION MANAGER: moved from #{orig_state} to #{cw.state}" }
+      end
+
+    end
+
+
+    # @param [AMQP-URL,Hash] primary_cnx valid argument to AMQP.connect to be
+    #   used as the primary connection.
+    #
+    # @param [AMQP-URL,Hash] secondary_cnx valid argument to AMQP.connect to be
+    #   used as the secondary connection.
+    #
+    # @option opts [Float] :primary_grace_time (5.0) how long we should wait
+    #   during startup for the primary to connect before giving up and just
+    #   using the secondary connection
+    #
+    def initialize(primary_opts, secondary_opts, opts={})
+      opts = opts.with_indifferent_access.reverse_merge(DEFAULT_OPTIONS)
+
+      @primary_opts   = primary_opts
+      @secondary_opts = secondary_opts
+
+      @primary_grace_time = opts[:primary_grace_time]
+
+      if @primary_opts.kind_of?(ConnectionWrapper)
+        @primary = @primary_opts
+        @primary.name ||= 'primary'
+      else
+        @primary = ConnectionWrapper.new(primary_opts, :name => 'primary')
+      end
+
+      if @secondary_opts.kind_of?(ConnectionWrapper)
+        @secondary = @secondary_opts
+        @secondary.name ||= 'secondary'
+      else
+        @secondary = ConnectionWrapper.new(secondary_opts, :name => 'secondary')
+      end
+
+      @active_connection = nil
+
+      super()
+    end
+
+    def start(*a, &b)
+      on_open(&b)
+      super(*a)
+    end
+
+    # connect to the broker, handle reconnections. barf if transition to
+    # :start_up state is not possible
+    def start!(&blk)
+      on_open(&b)
+      super(*a)
+    end
+
+    def close(*a, &b)
+      on_close(&b)
+      super(*a)
+    end
+
+    def close!(*a, &b)
+      on_close(&b)
+      super(*a)
+    end
+
+    def connection
+      @active_connection
+    end
+
+    def primary_connected?
+      primary and primary.connected?
+    end
+
+    def secondary_connected?
+      secondary and secondary.connected?
+    end
+
+    def usable?
+      failed_over? or degraded? or connected?
+    end
+
+    alias :useable? :usable?
+
+    protected
+      def do_connect_primary
+        @primary.on_failure { primary_failed }
+        @primary.on_open    { primary_connected }
+        @primary.connect
+      end
+
+      def do_connect_secondary
+        @secondary.on_failure { secondary_failed }
+        @secondary.on_open    { secondary_connected }
+        @secondary.connect
+      end
+
+      def do_close
+        logger.debug {"shutting down primary"}
+        @primary.close do
+          logger.debug {"shutting down secondary"}
+          @secondary.close do
+            logger.debug {"secondary shutdown, we're closed'"}
+            closed!
+          end
+        end
+      end
+
+      def start_primary_timer
+        @primary_timer ||= EM::Timer.new(@primary_grace_time) { primary_wait_timeout }
+      end
+
+      def clear_primary_timer!
+        if @primary_timer
+          @primary_timer.cancel
+          @primary_timer = nil
+        end
+      end
+
+      def fire_failure_deferred
+        reset_open_event
+        reset_failure_event.succeed
+      end
+
+      def fire_closed_callbacks
+        on_close.succeed
+      end
+
+      def enter_failed_over_state
+        clear_primary_timer!
+        @active_connection = @secondary
+        on_open.succeed
+      end
+
+      def become_usable!
+        clear_primary_timer!
+        @active_connection = @primary
+        on_open.succeed
+      end
+  end
+end
+

data/lib/who_can/connection_wrapper.rb

@@ -0,0 +1,210 @@
+module StateMachine
+  class StateMachine::Machine
+    include ::WhoCan::Logging
+  end
+end
+
+module WhoCan
+  # encapsulates the retry and failure detection logic. Used by ConnectionManager
+  class ConnectionWrapper
+    include Logging
+    include Deferred::Accessors
+
+    attr_reader :config, :amqp_session, :ekg
+
+    attr_accessor :name
+
+    # how long should we wait between reconnection attempts
+    attr_accessor :reconnect_delay, :shutdown_timeout
+
+    deferred_event :open, :failure, :close, :initial_connection_failure
+
+    state_machine :state, :initial => :new do
+      # moving from no-live-connection to attempting connection
+      after_transition [:new, :retrying] => :connecting,  :do => :do_connection
+
+      # callback for handling "failure to connect to the broker"
+      after_transition :connecting => :retrying, 
+        :do => :after_initial_connection_failure
+
+      # connection has been established
+      after_transition :connecting => :connected, :do => :after_connected
+
+      # if our connection failed in some way, be sure we shut down ekg
+      before_transition :connected => :retrying, :do => :shutdown_ekg
+      
+      # connection fails in some way, hook up retry logic
+      after_transition [:connecting, :connected] => :retrying, :do => [:fire_failure_callbacks, :do_retry]
+
+      # close has been requested
+      after_transition any => :closing,     :do => :do_close
+
+      # close has succeeded
+      after_transition [:closing, :new] => :closed, :do => :after_closed
+
+      # logging of state transitions
+      around_transition do |cw,block|
+        orig_state = cw.state
+        block.call
+        # logger.debug { "CONNECTION WRAPPER(#{cw.name}): moved from #{orig_state} to #{cw.state}" }
+      end
+
+      event :connect do
+        transition [:new, :retrying] => :connecting
+      end
+
+      event :connected do
+        transition :connecting => :connected
+      end
+
+      event :initial_connection_failure do
+        transition :connecting => :retrying
+      end
+
+      event :tcp_connection_failed do
+        transition :connected => :retrying
+      end
+
+      event :close do
+        transition [:connecting, :retrying, :connected] => :closing
+        transition :new => :closed
+      end
+
+      event :closed do
+        transition :closing => :closed
+      end
+    end
+
+
+    def initialize(config, opts={})
+      @config = config
+      @name             = opts[:name]
+      @reconnect_delay  = opts[:reconnect_delay]  || 5
+      @shutdown_timeout = opts[:shutdown_timeout] || 2.0
+
+      @ekg          = nil
+      @retry_count  = 0
+      @amqp_session = nil
+      @on_initial_connection_failure_cbs = []
+      super()
+    end
+
+    # XXX: figure out how to do this in a cleaner fashion
+    def connect(*a, &b)
+      on_open(&b)
+      super(*a)
+    end
+
+    def connect!(*a, &b)
+      on_open(&b)
+      super(*a)
+    end
+    
+    def close(*a, &b)
+      on_close(&b) # hook deferred into this event
+      super(*a)
+    end
+
+    def close!(*a, &b)
+      on_close(&b) # hook deferred into this event
+      super(*a)
+    end
+
+    protected
+      def do_connection
+        logger.info { "#{name} attempting connection to #{@config.inspect}" }
+
+        AMQP.connect(@config, :on_tcp_connection_failure => method(:initial_connection_failure!).to_proc) do |sess|
+          @ekg = Heartbeater::EKG.new(sess)
+
+          @ekg.on_heartbeat_failure { tcp_connection_failed }
+
+          @ekg.start!
+          @amqp_session = sess
+
+          connected
+        end
+      end
+
+      def do_close
+        @retry_timer.cancel if @retry_timer
+
+        logger.debug { "wrapper(#{name}): do_close called" }
+        
+        if @amqp_session || @ekg
+          logger.debug { "wrapper(#{name}): closing ekg!" }
+          shutdown_ekg { closed }
+        else
+          closed
+        end
+      end
+
+      def fire_failure_callbacks
+        return if closing? or closed?
+        logger.debug { "wrapper(#{name}): fire_failure_callbacks" }
+        reset_open_event
+        reset_failure_event.succeed
+      end
+
+      def shutdown_ekg(&blk)
+        Deferred::Default.new.tap do |my_dfr|
+          if @ekg
+            amqp_session, @amqp_session = @amqp_session, nil
+
+            my_dfr.timeout(@shutdown_timeout) # we need to ensure that we will time out
+            my_dfr.ensure_that(&blk) if blk
+
+            if amqp_session
+              @ekg.on_shutdown.ensure_that do
+                logger.debug { "wrapper(#{name}): ekg shutdown completed" }
+
+                amqp_session.on_closed { my_dfr.succeed }
+
+                amqp_session.close
+              end
+            end
+            
+            logger.debug { "wrapper(#{name}): shutting down ekg" }
+
+            @ekg.on_shutdown.errback do |e|
+              logger.debug { "wrapper(#{name}): ekg shutdown timed out!" }
+            end
+
+            @ekg.on_shutdown.timeout(@shutdown_timeout, TimeoutError) # ensure that ekg shutdown won't hang
+            @ekg.shutdown! 
+            @ekg = nil
+          else
+            my_dfr.succeed
+          end
+        end
+      end
+
+      def after_connected
+        on_open.succeed
+      end
+
+      def after_closed
+        on_close.succeed
+      end
+
+      def after_initial_connection_failure
+        logger.warn { "wrapper(#{name}): connection to #{@config.inspect} failed, this is either a misconfiguration or the server is down" }
+        logger.warn { "wrapper(#{name}): retrying after a delay of #{reconnect_delay}s" }
+
+        reset_initial_connection_failure_event.succeed
+      end
+
+      def do_retry
+        return if closing? or closed?
+
+        logger.debug { "wrapper(#{name}): do_retry setting up reconnection timer" }
+
+        @retry_timer = EM::Timer.new(reconnect_delay) do
+          @retry_count += 1
+          logger.debug { "wrapper(#{name}): retry attempt: #{@retry_count}" }
+          connect!
+        end
+      end
+  end
+end
+