nerve_pharmeasy 0.7.0

data/lib/nerve/reporter/base.rb

@@ -0,0 +1,61 @@
+
+class Nerve::Reporter
+  class Base
+    include Nerve::Utils
+    include Nerve::Logging
+
+    def initialize(opts)
+    end
+
+    def start
+    end
+
+    def stop
+    end
+
+    def report_up
+    end
+
+    def report_down
+    end
+
+    def ping?
+    end
+
+    def get_service_data(service)
+      %w{instance_id host port}.each do |required|
+        raise ArgumentError, "missing required argument #{required} for new service watcher" unless service[required]
+      end
+      d = {
+        'host' => service['host'],
+        'port' => service['port'],
+        'name' => service['instance_id']
+      }
+
+      # Weight is optional, but it should be well formed if supplied
+      if service.has_key?('weight')
+        if service['weight'].to_i >= 0 and "#{service['weight']}".match /^\d+$/
+          d['weight'] = service['weight'].to_i
+        else
+          raise ArgumentError, "invalid 'weight' argument in service data: #{service.inspect}"
+        end
+      end
+
+      if service.has_key?('haproxy_server_options')
+        d['haproxy_server_options'] = service['haproxy_server_options']
+      end
+
+      if service.has_key?('labels')
+        d['labels'] = service['labels']
+      end
+      d
+    end
+
+    protected
+    def parse_data(data)
+      return data if data.class == String
+      return data.to_json
+    end
+  end
+end
+

data/lib/nerve/reporter/etcd.rb

@@ -0,0 +1,73 @@
+require 'nerve/reporter/base'
+require 'etcd'
+
+class Nerve::Reporter
+  class Etcd < Base
+    def initialize(service)
+      raise ArgumentError, "missing required argument etcd_host for new service watcher" unless service['etcd_host']
+      @host = service['etcd_host']
+      @port = service['etcd_port'] || 4003
+      path = service['etcd_path'] || '/'
+      @path = path.split('/').push(service['instance_id']).join('/')
+      @data = parse_data(get_service_data(service))
+      @key = nil
+      @ttl = (service['check_interval'] || 0.5) * 5
+      @ttl = @ttl.ceil
+    end
+
+    def start()
+      log.info "nerve: connecting to etcd at #{@host}:#{@port}"
+      @etcd = ::Etcd.client(:host => @host, :port => @port)
+      log.info "nerve: successfully created etcd connection to #{@host}:#{@port}"
+    end
+
+    def stop()
+       report_down
+       @etcd = nil
+    end
+
+    def report_up()
+      etcd_save
+    end
+
+    def report_down
+      etcd_delete
+    end
+
+    def ping?
+      # we get a ping every check_interval.
+      if @key
+        # we have made a key: save it to prevent the TTL from expiring.
+        etcd_save
+      else
+        # we haven't created a key, so just frob the etcd API to assure that
+        # it's alive.
+        @etcd.leader
+      end
+    end
+
+    private
+
+    def etcd_delete
+      return unless @etcd and @key
+      begin
+        @etcd.delete(@key)
+      rescue ::Etcd::NotFile
+      rescue Errno::ECONNREFUSED
+      end
+    end
+
+    def etcd_create
+      # we use create_in_order to create a unique key under our path,
+      # permitting multiple registrations from the same instance_id.
+      @key = @etcd.create_in_order(@path, :value => @data, :ttl => @ttl).key
+      log.info "registered etcd key #{@key} with value #{@data}, TTL #{@ttl}"
+    end
+
+    def etcd_save
+      return etcd_create unless @key
+      @etcd.set(@key, :value => @data, :ttl => @ttl)
+    end
+  end
+end
+

data/lib/nerve/reporter/zookeeper.rb

@@ -0,0 +1,101 @@
+require 'nerve/reporter/base'
+require 'thread'
+require 'zk'
+
+
+class Nerve::Reporter
+  class Zookeeper < Base
+    @@zk_pool = {}
+    @@zk_pool_count = {}
+    @@zk_pool_lock = Mutex.new
+
+    def initialize(service)
+      %w{zk_hosts zk_path}.each do |required|
+        raise ArgumentError, "missing required argument #{required} for new service watcher" unless service[required]
+      end
+      # Since we pool we get one connection per zookeeper cluster
+      @zk_connection_string = service['zk_hosts'].sort.join(',')
+      @data = parse_data(get_service_data(service))
+
+      @zk_path = service['zk_path']
+      @key_prefix = @zk_path + "/#{service['instance_id']}_"
+      @full_key = nil
+    end
+
+    def start()
+      log.info "nerve: waiting to connect to zookeeper to #{@zk_connection_string}"
+      # Ensure that all Zookeeper reporters re-use a single zookeeper
+      # connection to any given set of zk hosts.
+      @@zk_pool_lock.synchronize {
+        unless @@zk_pool.has_key?(@zk_connection_string)
+          log.info "nerve: creating pooled connection to #{@zk_connection_string}"
+          @@zk_pool[@zk_connection_string] = ZK.new(@zk_connection_string, :timeout => 5)
+          @@zk_pool_count[@zk_connection_string] = 1
+          log.info "nerve: successfully created zk connection to #{@zk_connection_string}"
+        else
+          @@zk_pool_count[@zk_connection_string] += 1
+          log.info "nerve: re-using existing zookeeper connection to #{@zk_connection_string}"
+        end
+        @zk = @@zk_pool[@zk_connection_string]
+        log.info "nerve: retrieved zk connection to #{@zk_connection_string}"
+      }
+    end
+
+    def stop()
+      log.info "nerve: removing zk node at #{@full_key}" if @full_key
+      begin
+        report_down
+      ensure
+        @@zk_pool_lock.synchronize {
+          @@zk_pool_count[@zk_connection_string] -= 1
+          # Last thread to use the connection closes it
+          if @@zk_pool_count[@zk_connection_string] == 0
+            log.info "nerve: closing zk connection to #{@zk_connection_string}"
+            begin
+              @zk.close!
+            ensure
+              @@zk_pool.delete(@zk_connection_string)
+            end
+          end
+        }
+      end
+    end
+
+    def report_up()
+      zk_save
+    end
+
+    def report_down
+      zk_delete
+    end
+
+    def ping?
+      return @zk.connected? && @zk.exists?(@full_key || '/')
+    end
+
+    private
+
+    def zk_delete
+      if @full_key
+        @zk.delete(@full_key, :ignore => :no_node)
+        @full_key = nil
+      end
+    end
+
+    def zk_create
+      @zk.mkdir_p(@zk_path)
+      @full_key = @zk.create(@key_prefix, :data => @data, :mode => :ephemeral_sequential)
+    end
+
+    def zk_save
+      return zk_create unless @full_key
+
+      begin
+        @zk.set(@full_key, @data)
+      rescue ZK::Exceptions::NoNode
+        zk_create
+      end
+    end
+  end
+end
+

data/lib/nerve/reporter.rb

@@ -0,0 +1,18 @@
+require 'nerve/utils'
+require 'nerve/log'
+require 'nerve/reporter/base'
+
+module Nerve
+  class Reporter
+    def self.new_from_service(service)
+      type = service['reporter_type'] || 'zookeeper'
+      reporter = begin
+        require "nerve/reporter/#{type.downcase}"
+        self.const_get(type.downcase.capitalize)
+      rescue Exception => e
+        raise ArgumentError, "specified a reporter_type of #{type}, which could not be found: #{e}"
+      end
+      reporter.new(service)
+    end
+  end
+end

data/lib/nerve/ring_buffer.rb

@@ -0,0 +1,30 @@
+module Nerve
+  class RingBuffer < Array
+    alias_method :array_push, :push
+    alias_method :array_element, :[]
+    
+    def initialize( size )
+      @ring_size = size.to_i
+      super( @ring_size )
+    end
+    
+    def average
+      self.inject(0.0) { |sum, el| sum + el } / self.size
+    end
+    
+    def push( element )
+      if length == @ring_size
+        shift # loose element
+      end
+      array_push element
+    end
+    
+    # Access elements in the RingBuffer
+    #
+    # offset will be typically negative!
+    #
+    def []( offset = 0 )
+      return self.array_element( - 1 + offset )
+    end
+  end
+end

data/lib/nerve/service_watcher/base.rb

@@ -0,0 +1,65 @@
+require 'nerve/ring_buffer'
+
+module Nerve
+  module ServiceCheck
+    class BaseServiceCheck
+      include Utils
+      include Logging
+
+      def initialize(opts={})
+        @timeout = opts['timeout'] ? opts['timeout'].to_f : 0.1
+        @rise    = opts['rise']    ? opts['rise'].to_i    : 1
+        @fall    = opts['fall']    ? opts['fall'].to_i    : 1
+        @name    = opts['name']    ? opts['name']         : "undefined"
+
+        @check_buffer = RingBuffer.new([@rise, @fall].max)
+        @last_result = nil
+      end
+
+      def up?
+        # do the check
+        check_result = !!catch_errors do
+          check
+        end
+
+        # this is the first check -- initialize buffer
+        if @last_result == nil
+          @last_result = check_result
+          @check_buffer.size.times {@check_buffer.push check_result}
+          log.info "nerve: service check #{@name} initial check returned #{check_result}"
+        end
+
+        log.debug "nerve: service check #{@name} returned #{check_result}"
+        @check_buffer.push(check_result)
+
+        # we've failed if the last @fall times are false
+        unless @check_buffer.last(@fall).reduce(:|)
+          log.info "nerve: service check #{@name} transitions to down after #{@fall} failures" if @last_result
+          @last_result = false
+        end
+
+        # we've succeeded if the last @rise times is true
+        if @check_buffer.last(@rise).reduce(:&)
+          log.info "nerve: service check #{@name} transitions to up after #{@rise} successes" unless @last_result
+          @last_result = true
+        end
+
+        # otherwise return the last result
+        return @last_result
+      end
+
+      def catch_errors(&block)
+        begin
+          return yield
+        rescue Object => error
+          log.info "nerve: service check #{@name} got error #{error.inspect}"
+          return false
+        end
+      end
+    end
+
+    CHECKS ||= {}
+    CHECKS['base'] = BaseServiceCheck
+  end
+end
+

data/lib/nerve/service_watcher/http.rb

@@ -0,0 +1,70 @@
+require 'nerve/service_watcher/base'
+
+module Nerve
+  module ServiceCheck
+    require 'net/http'
+
+    class HttpServiceCheck < BaseServiceCheck
+      def initialize(opts={})
+        super
+
+        %w{port uri}.each do |required|
+          raise ArgumentError, "missing required argument #{required} in http check" unless
+            opts[required]
+          instance_variable_set("@#{required}",opts[required])
+        end
+
+        @host        = opts['host'] || '127.0.0.1'
+        @ssl         = opts['ssl']  || false
+
+        @read_timeout = opts['read_timeout'] || @timeout
+        @open_timeout = opts['open_timeout'] || 0.2
+        @ssl_timeout  = opts['ssl_timeout']  || 0.2
+
+        @headers     = opts['headers'] || {}
+
+        @expect      = opts['expect']
+
+        @name        = "http-#{@host}:#{@port}#{@uri}"
+      end
+
+      def check
+        log.debug "running health check #{@name}"
+
+        connection = get_connection
+        response = connection.get(@uri, @headers)
+        code = response.code.to_i
+        body = response.body
+
+        # Any 2xx or 3xx code should be considered healthy. This is standard
+        # practice in HAProxy, nginx, etc ...
+        if code >= 200 and code < 400 and (@expect == nil || body.include?(@expect))
+          log.debug "nerve: check #{@name} got response code #{code} with body \"#{body}\""
+          return true
+        else
+          log.warn "nerve: check #{@name} got response code #{code} with body \"#{body}\""
+          return false
+        end
+      end
+
+      private
+      def get_connection
+        con = Net::HTTP.new(@host, @port)
+        con.read_timeout = @read_timeout
+        con.open_timeout = @open_timeout
+
+        if @ssl
+          con.use_ssl = true
+          con.ssl_timeout = @ssl_timeout
+          con.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+
+        return con
+      end
+
+    end
+
+    CHECKS ||= {}
+    CHECKS['http'] = HttpServiceCheck
+  end
+end

data/lib/nerve/service_watcher/rabbitmq.rb

@@ -0,0 +1,68 @@
+require 'nerve/service_watcher/base'
+require 'bunny'
+
+module Nerve
+  module ServiceCheck
+    class RabbitMQServiceCheck < BaseServiceCheck
+      require 'socket'
+      include Socket::Constants
+
+      def initialize(opts={})
+        super
+
+        raise ArgumentError, "missing required argument 'port' in rabbitmq check" unless opts['port']
+
+        @port = opts['port']
+        @host = opts['host']     || '127.0.0.1'
+        @user = opts['username'] || 'guest'
+        @pass = opts['password'] || 'guest'
+      end
+
+      def check
+        # the idea for this check was taken from the one in rabbitmq management
+        #  -- the aliveness_test:
+        # https://github.com/rabbitmq/rabbitmq-management/blob/9a8e3d1ab5144e3f6a1cb9a4639eb738713b926d/src/rabbit_mgmt_wm_aliveness_test.erl
+        log.debug "nerve: running rabbitmq health check #{@name}"
+
+        conn = Bunny.new(
+          :host => @host,
+          :port => @port,
+          :user => @user,
+          :pass => @pass,
+          :log_file => STDERR,
+          :continuation_timeout => @timeout,
+          :automatically_recover => false,
+          :heartbeat => false,
+          :threaded => false
+        )
+
+        begin
+          conn.start
+          ch = conn.create_channel
+
+          # create a queue, publish to it
+          log.debug "nerve: publishing to rabbitmq"
+          ch.queue('nerve')
+          ch.basic_publish('nerve test message', '', 'nerve', :mandatory => true, :expiration => 2 * 1000)
+
+          # read and ack the message
+          log.debug "nerve: consuming from rabbitmq"
+          delivery_info, properties, payload = ch.basic_get('nerve', :ack => true)
+
+          if payload
+            ch.acknowledge(delivery_info.delivery_tag)
+            return true
+          else
+            log.debug "nerve: rabbitmq consumption returned no payload"
+            return false
+          end
+        ensure
+          conn.close
+        end
+      end
+    end
+
+    CHECKS ||= {}
+    CHECKS['rabbitmq'] = RabbitMQServiceCheck
+  end
+end

data/lib/nerve/service_watcher/tcp.rb

@@ -0,0 +1,56 @@
+require 'nerve/service_watcher/base'
+
+module Nerve
+  module ServiceCheck
+    class TcpServiceCheck < BaseServiceCheck
+      require 'socket'
+      include Socket::Constants
+
+      def initialize(opts={})
+        super
+
+        raise ArgumentError, "missing required argument 'port' in tcp check" unless opts['port']
+
+        @port = opts['port']
+        @host = opts['host'] || '127.0.0.1'
+
+        @address = Socket.sockaddr_in(@port, @host)
+      end
+
+      def check
+        log.debug "nerve: running TCP health check #{@name}"
+
+        # create a TCP socket
+        socket = Socket.new(AF_INET, SOCK_STREAM, 0)
+
+        begin
+          # open a non-blocking connection
+          socket.connect_nonblock(@address)
+        rescue Errno::EINPROGRESS
+          # opening a non-blocking socket will usually raise
+          # this exception. it's just connect returning immediately,
+          # so it's not really an exception, but ruby makes it into
+          # one. if we got here, we are now free to wait until the timeout
+          # expires for the socket to be writeable
+          IO.select(nil, [socket], nil, @timeout)
+
+          # we should be connected now; allow any other exception through
+          begin
+            socket.connect_nonblock(@address)
+          rescue Errno::EISCONN
+            return true
+          end
+        else
+          # we managed to connect REALLY REALLY FAST
+          log.debug "nerve: connected to non-blocking socket without an exception"
+          return true
+        ensure
+          socket.close
+        end
+      end
+    end
+
+    CHECKS ||= {}
+    CHECKS['tcp'] = TcpServiceCheck
+  end
+end

data/lib/nerve/service_watcher.rb

@@ -0,0 +1,152 @@
+require 'nerve/service_watcher/tcp'
+require 'nerve/service_watcher/http'
+require 'nerve/service_watcher/rabbitmq'
+
+module Nerve
+  class ServiceWatcher
+    include Utils
+    include Logging
+
+    attr_reader :was_up
+
+    def initialize(service={})
+      log.debug "nerve: creating service watcher object"
+
+      # check that we have all of the required arguments
+      %w{name instance_id host port}.each do |required|
+        raise ArgumentError, "missing required argument #{required} for new service watcher" unless service[required]
+      end
+
+      @name = service['name']
+
+      # configure the reporter, which we use for reporting status to the registry 
+      @reporter = Reporter.new_from_service(service)
+
+      # instantiate the checks for this service
+      @service_checks = []
+      service['checks'] ||= []
+      service['checks'].each do |check|
+        # checks inherit attributes from the service overall
+        check['host'] ||= service['host']
+        check['port'] ||= service['port']
+
+        # generate a nice readable name for each check
+        check['name'] ||= "#{@name} #{check['type']}-#{check['host']}:#{check['port']}"
+
+        # make sure a type is set
+        check['type'] ||= "undefined"
+
+        # require a 3rd-party module if necessary for external checkers
+        unless ServiceCheck::CHECKS[check['type']]
+          m = check['module'] ? check['module'] : "nerve-watcher-#{check['type']}"
+          require m
+        end
+
+        # instantiate the check object
+        service_check_class = ServiceCheck::CHECKS[check['type']]
+        if service_check_class.nil?
+          raise ArgumentError,
+            "invalid service check type #{check['type']}; valid types: #{ServiceCheck::CHECKS.keys.join(',')}"
+        end
+
+        # save the check object
+        @service_checks << service_check_class.new(check)
+      end
+
+      # how often do we initiate service checks?
+      @check_interval = service['check_interval'] || 0.5
+
+      # force an initial report on startup
+      @was_up = nil
+
+      # when this watcher is started it will store the
+      # thread here
+      @run_thread = nil
+      @should_finish = false
+
+      log.debug "nerve: created service watcher for #{@name} with #{@service_checks.size} checks"
+    end
+
+    def start()
+      unless @run_thread
+        @run_thread = Thread.new { self.run() }
+      else
+        log.error "nerve: tried to double start a watcher"
+      end
+    end
+
+    def stop()
+      log.info "nerve: stopping service watch #{@name}"
+      @should_finish = true
+      return true if @run_thread.nil?
+
+      unclean_shutdown = @run_thread.join(10).nil?
+      if unclean_shutdown
+        log.error "nerve: unclean shutdown of #{@name}, killing thread"
+        Thread.kill(@run_thread)
+      end
+      @run_thread = nil
+      !unclean_shutdown
+    end
+
+    def alive?()
+      !@run_thread.nil? && @run_thread.alive?
+    end
+
+    def run()
+      log.info "nerve: starting service watch #{@name}"
+      @reporter.start()
+
+      until watcher_should_exit?
+        check_and_report
+
+        # wait to run more checks but make sure to exit if $EXIT
+        # we avoid sleeping for the entire check interval at once
+        # so that nerve can exit promptly if required
+        responsive_sleep (@check_interval) { watcher_should_exit? }
+      end
+    rescue StandardError => e
+      log.error "nerve: error in service watcher #{@name}: #{e.inspect}"
+      raise e
+    ensure
+      log.info "nerve: stopping reporter for #{@name}"
+      @reporter.stop
+    end
+
+    def check_and_report
+      if !@reporter.ping?
+        # If the reporter can't ping, then we do not know the status
+        # and must force a new report.
+        @was_up = nil
+      end
+
+      # what is the status of the service?
+      is_up = check?
+      log.debug "nerve: current service status for #{@name} is #{is_up.inspect}"
+
+      if is_up != @was_up
+        if is_up
+          @reporter.report_up
+          log.info "nerve: service #{@name} is now up"
+        else
+          @reporter.report_down
+          log.warn "nerve: service #{@name} is now down"
+        end
+        @was_up = is_up
+      end
+    end
+
+    def check?
+      @service_checks.each do |check|
+        return false unless check.up?
+      end
+      return true
+    end
+
+    private
+    def watcher_should_exit?
+      $EXIT || @should_finish
+    end
+
+  end
+end