nerve_pharmeasy 0.7.0

data/lib/nerve/utils.rb

@@ -0,0 +1,17 @@
+module Nerve
+  module Utils
+    def safe_run(command)
+      res = `#{command}`.chomp
+      raise "command '#{command}' failed to run:\n#{res}" unless $?.success?
+    end
+
+    def responsive_sleep(seconds, tick=1, &should_exit)
+      nap_time = seconds
+      while nap_time > 0
+        break if (should_exit && should_exit.call)
+        sleep [nap_time, tick].min
+        nap_time -= tick
+      end
+    end
+  end
+end

data/lib/nerve/version.rb

@@ -0,0 +1,3 @@
+module Nerve
+  VERSION = "0.8.0"
+end

data/lib/nerve.rb

@@ -0,0 +1,249 @@
+require 'fileutils'
+require 'logger'
+require 'json'
+require 'timeout'
+require 'socket'
+
+require 'nerve/version'
+require 'nerve/utils'
+require 'nerve/log'
+require 'nerve/ring_buffer'
+require 'nerve/reporter'
+require 'nerve/service_watcher'
+
+require 'nerve/configuration_manager'
+
+module Nerve
+  class Nerve
+    include Logging
+    include Utils
+
+    MAIN_LOOP_SLEEP_S = 10.freeze
+    LAUNCH_WAIT_FOR_REPORT_S = 30.freeze
+
+    def initialize(config_manager)
+      log.info 'nerve: setting up!'
+      @config = config_manager
+
+      # set global variable for exit signal
+      $EXIT = false
+
+      # State of currently running watchers according to Nerve
+      @watchers = {}
+      @watcher_versions = {}
+
+      # instance_id, heartbeat_path, and watchers_desired are populated by
+      # load_config! in the main loop from the configuration source
+      @instance_id = nil
+      @heartbeat_path = nil
+      @watchers_desired = {}
+
+      # Flag to indicate a config reload is required by the main loop
+      # This decoupling is required for gracefully reloading config on SIGHUP
+      # as one should do as little as possible in a signal handler
+      @config_to_load = true
+
+      Signal.trap("HUP") do
+        @config_to_load = true
+      end
+
+      log.debug 'nerve: completed init'
+    end
+
+    def load_config!
+      log.info 'nerve: loading config'
+      @config_to_load = false
+      log.info @config_manager.class.name
+
+      @config.each do|key, value|
+        log.info key
+        log.info value
+      end
+
+      services = @config["services"]
+
+      services.each do|service, data|
+        log.info "service:"+service
+        host = data["host"]
+        log.info "host:"+data["host"]
+        if host == "localhost"
+            log.info local_ip
+             data["host"]=local_ip
+        end
+      end
+      #kedar @config_manager.reload!
+      #kedar config = @config_manager.config
+
+      @config_manager = ConfigurationManager.new()
+      @config_manager.setConfig(@config)
+
+
+      # required options
+      log.debug 'nerve: checking for required inputs'
+      %w{instance_id services}.each do |required|
+        raise ArgumentError, "you need to specify required argument #{required}" unless @config[required]
+      end
+      @instance_id = @config['instance_id']
+      @watchers_desired = @config['services']
+      @heartbeat_path = @config['heartbeat_path']
+    end
+
+    def local_ip
+      orig = Socket.do_not_reverse_lookup
+      Socket.do_not_reverse_lookup =true # turn off reverse DNS resolution temporarily
+      UDPSocket.open do |s|
+        s.connect '64.233.187.99', 1 #google
+        s.addr.last
+      end
+    ensure
+      Socket.do_not_reverse_lookup = orig
+    end
+
+    def run
+      log.info 'nerve: starting main run loop'
+      begin
+        until $EXIT
+          # Check if configuration needs to be reloaded and reconcile any new
+          # configuration of watchers with old configuration
+          if @config_to_load
+            load_config!
+
+            # Reap undesired service watchers
+            services_to_reap = @watchers.select{ |name, _|
+              !@watchers_desired.has_key?(name)
+            }.keys()
+
+            unless services_to_reap.empty?
+              log.info "nerve: reaping old watchers: #{services_to_reap}"
+              services_to_reap.each do |name|
+                  reap_watcher(name)
+              end
+            end
+
+            # Start new desired service watchers
+            services_to_launch = @watchers_desired.select{ |name, _|
+              !@watchers.has_key?(name)
+            }.keys()
+
+            unless services_to_launch.empty?
+              log.info "nerve: launching new watchers: #{services_to_launch}"
+              services_to_launch.each do |name|
+                launch_watcher(name, @watchers_desired[name])
+              end
+            end
+
+            # Detect and update existing service watchers which are in both
+            # the currently running state and the desired (config) watcher
+            # state but have different configurations
+            services_to_update = @watchers.select { |name, _|
+              @watchers_desired.has_key?(name) &&
+              merged_config(@watchers_desired[name], name).hash != @watcher_versions[name]
+            }.keys()
+
+            services_to_update.each do |name|
+              log.info "nerve: detected new config for #{name}"
+              # Keep the old watcher running until the replacement is launched
+              # This keeps the service registered while we change it over
+              # This also keeps connection pools active across diffs
+              temp_name = "#{name}_#{@watcher_versions[name]}"
+              @watchers[temp_name] = @watchers.delete(name)
+              @watcher_versions[temp_name] = @watcher_versions.delete(name)
+              log.info "nerve: launching new watcher for #{name}"
+              launch_watcher(name, @watchers_desired[name], :wait => true)
+              log.info "nerve: reaping old watcher #{temp_name}"
+              reap_watcher(temp_name)
+            end
+          end
+
+          # If this was a configuration check, bail out now
+          if @config_manager.options[:check_config]
+            log.info 'nerve: configuration check succeeded, exiting immediately'
+            break
+          end
+
+          # Check that watchers are still alive, auto-remediate if they
+          # are not. Sometimes zookeeper flakes out or connections are lost to
+          # remote datacenter zookeeper clusters, failing is not an option
+          relaunch = []
+          @watchers.each do |name, watcher|
+            unless watcher.alive?
+              relaunch << name
+            end
+          end
+
+          relaunch.each do |name|
+            begin
+              log.warn "nerve: watcher #{name} not alive; reaping and relaunching"
+              reap_watcher(name)
+            rescue => e
+              log.warn "nerve: could not reap #{name}, got #{e.inspect}"
+            end
+            launch_watcher(name, @watchers_desired[name])
+          end
+
+          # Indicate we've made progress
+          heartbeat()
+
+          responsive_sleep(MAIN_LOOP_SLEEP_S) { @config_to_load || $EXIT }
+        end
+      rescue => e
+        log.error "nerve: encountered unexpected exception #{e.inspect} in main thread"
+        raise e
+      ensure
+        $EXIT = true
+        log.warn 'nerve: reaping all watchers'
+        @watchers.each do |name, _|
+          reap_watcher(name)
+        end
+      end
+
+      log.info 'nerve: exiting'
+    ensure
+      $EXIT = true
+    end
+
+    def heartbeat
+      unless @heartbeat_path.nil?
+        FileUtils.touch(@heartbeat_path)
+      end
+      log.debug 'nerve: heartbeat'
+    end
+
+    def merged_config(config, name)
+      # Get a deep copy so sub-hashes are properly handled
+      deep_copy = Marshal.load(Marshal.dump(config))
+      return deep_copy.merge({'instance_id' => @instance_id, 'name' => name})
+    end
+
+    def launch_watcher(name, config, opts = {})
+      wait = opts[:wait] || false
+
+      watcher_config = merged_config(config, name)
+      # The ServiceWatcher may mutate the configs, so record the version before
+      # passing the config to the ServiceWatcher
+      @watcher_versions[name] = watcher_config.hash
+
+      watcher = ServiceWatcher.new(watcher_config)
+      unless @config_manager.options[:check_config]
+        log.debug "nerve: launching service watcher #{name}"
+        watcher.start()
+        @watchers[name] = watcher
+        if wait
+          log.info "nerve: waiting for watcher thread #{name} to report"
+          responsive_sleep(LAUNCH_WAIT_FOR_REPORT_S) { !watcher.was_up.nil? || $EXIT }
+          log.info "nerve: watcher thread #{name} has reported!"
+        end
+      else
+        log.info "nerve: not launching #{name} due to --check-config option"
+      end
+    end
+
+    def reap_watcher(name)
+      watcher = @watchers.delete(name)
+      @watcher_versions.delete(name)
+      shutdown_status = watcher.stop()
+      log.info "nerve: stopped #{name}, clean shutdown? #{shutdown_status}"
+      shutdown_status
+    end
+  end
+end

data/nerve.conf.json

@@ -0,0 +1,23 @@
+{
+  "instance_id": "mymachine",
+  "services": {
+    "nodejs": {
+      "host": "localhost",
+      "port": 3000,
+      "reporter_type": "zookeeper",
+      "zk_hosts": ["localhost:2181"],
+      "zk_path": "/nerve/services/your_http_service/services",
+      "check_interval": 2,
+      "checks": [
+        {
+          "type": "http",
+          "uri": "/health",
+          "timeout": 0.2,
+          "rise": 3,
+          "fall": 2
+        }
+      ]
+    }
+  
+  }
+}

data/nerve.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'nerve/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "nerve"
+  gem.version       = Nerve::VERSION
+  gem.authors       = ["Martin Rhoads", "Igor Serebryany", "Pierre Carrier"]
+  gem.email         = ["martin.rhoads@airbnb.com", "igor.serebryany@airbnb.com"]
+  gem.description   = "Nerve is a service registration daemon. It performs health "\
+                      "checks on your service and then publishes success or failure "\
+                      "into one of several registries (currently, zookeeper or etcd). "\
+                      "Nerve is half or SmartStack, and is designed to be operated "\
+                      "along with Synapse to provide a full service discovery framework"
+  gem.summary       = %q{A service registration daemon}
+  gem.homepage      = "https://github.com/airbnb/nerve"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "json"
+  gem.add_runtime_dependency "zk", "~> 1.9.2"
+  gem.add_runtime_dependency "bunny", "= 1.1.0"
+  gem.add_runtime_dependency "etcd", "~> 0.2.3"
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec", "~> 3.1.0"
+  gem.add_development_dependency "factory_girl"
+  gem.add_development_dependency "pry"
+end

data/spec/configuration_manager_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+require 'nerve/configuration_manager'
+
+describe Nerve::ConfigurationManager do
+  describe 'parsing config' do
+    let(:config_manager) { Nerve::ConfigurationManager.new() }
+    let(:nerve_config) { "#{File.dirname(__FILE__)}/../example/nerve.conf.json" }
+    let(:nerve_instance_id) { 'testid' }
+
+    it 'parses valid options' do
+      allow(config_manager).to receive(:parse_options_from_argv!) { {
+        :config => nerve_config,
+        :instance_id => nerve_instance_id,
+        :check_config => false
+      } }
+
+      expect{config_manager.reload!}.to raise_error(RuntimeError)
+      expect(config_manager.parse_options!).to eql({
+        :config => nerve_config,
+        :instance_id => nerve_instance_id,
+        :check_config => false
+      })
+      expect{config_manager.reload!}.not_to raise_error
+      expect(config_manager.config.keys()).to include('instance_id', 'services')
+      expect(config_manager.config['services'].keys()).to contain_exactly(
+        'your_http_service', 'your_tcp_service', 'rabbitmq_service',
+        'etcd_service1', 'zookeeper_service1'
+      )
+    end
+  end
+end

data/spec/example_services_spec.rb

@@ -0,0 +1,42 @@
+require 'json'
+require 'nerve/reporter'
+require 'nerve/reporter/base'
+require 'nerve/service_watcher'
+
+class Nerve::Reporter::Base
+  attr_reader :data
+end
+
+describe "example services are valid" do
+  Dir.foreach("#{File.dirname(__FILE__)}/../example/nerve_services") do |item|
+    next if item == '.' or item == '..'
+    service_data = JSON.parse(IO.read("#{File.dirname(__FILE__)}/../example/nerve_services/#{item}"))
+    service_data['name'] = item.gsub(/\.json$/, '')
+    service_data['instance_id'] = '1'
+
+    context "when #{item} can be initialized as a valid reporter" do
+      it 'creates a valid reporter in new_from_service' do
+        reporter = nil
+        expect { reporter = Nerve::Reporter.new_from_service(service_data) }.to_not raise_error()
+        expect(reporter.is_a?(Nerve::Reporter::Base)).to eql(true)
+      end
+      it 'saves the weight data' do
+        expect(JSON.parse(Nerve::Reporter.new_from_service(service_data).data)['weight']).to eql(2)
+      end
+      it 'saves the labels data' do
+        labels = {'az' => 'us-west-1'}
+        service_data['labels'] = labels
+        expect(JSON.parse(Nerve::Reporter.new_from_service(service_data).data)['labels']).to eq(labels)
+      end
+    end
+
+    context "when #{item} can be initialized as a valid service watcher" do
+      it "creates a valid service watcher for #{item}" do
+        watcher = nil
+        expect { watcher = Nerve::ServiceWatcher.new(service_data) }.to_not raise_error()
+        expect(watcher.is_a?(Nerve::ServiceWatcher)).to eql(true)
+      end
+    end
+  end
+end
+

data/spec/factories/check.rb

@@ -0,0 +1,16 @@
+FactoryGirl.define do
+  factory :check, :class => Hash do
+    type       'base'
+    timeout    0.2
+    rise       3
+    fall       2
+
+    trait :http do
+      type   'http'
+      uri    '/health'
+    end
+
+    initialize_with { Hash[attributes.map{|k,v| [k.to_s,v]}] }
+    to_create {}
+  end
+end

data/spec/factories/service.rb

@@ -0,0 +1,26 @@
+FactoryGirl.define do
+  factory :service, :class => Hash do
+    sequence(:name)    {|n| "service_check_#{n}"}
+    instance_id        'public_hostname.example.com'
+    host               'localhost'
+    port               3000
+    reporter_type      'base'
+    checks             { create_list(:check, checks_count) }
+    check_interval     nil
+
+    trait :zookeeper do
+      reporter_type    'zookeeper'
+      zk_hosts         ['localhost:2181']
+      zk_path          { "/nerve/services/#{name}/services" }
+    end
+
+    # set up some service checks
+    transient do
+      checks_count 1
+    end
+
+    # thanks to https://stackoverflow.com/questions/10032760
+    initialize_with { Hash[attributes.map{|k,v| [k.to_s,v]}] }
+    to_create {}
+  end
+end

data/spec/lib/nerve/reporter_etcd_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+require 'nerve/reporter/etcd'
+
+describe Nerve::Reporter::Etcd do
+  let(:subject) { {
+      'etcd_host' => 'etcdhost1',
+      'etcd_port' => 4001,
+      'etcd_path' => '/path',
+      'instance_id' => 'instance_id',
+      'host' => 'host',
+      'port' => 'port'
+    }
+  }
+  it 'actually constructs an instance' do
+    expect(Nerve::Reporter::Etcd.new(subject).is_a?(Nerve::Reporter::Etcd)).to eql(true)
+  end
+end
+

data/spec/lib/nerve/reporter_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+require 'nerve/reporter/zookeeper'
+
+describe Nerve::Reporter do
+  let(:subject) { {
+      'zk_hosts' => ['zkhost1', 'zkhost2'],
+      'zk_path' => 'zk_path',
+      'instance_id' => 'instance_id',
+      'host' => 'host',
+      'port' => 'port'
+    }
+  }
+  it 'can new_from_service' do
+    expect(Nerve::Reporter::Zookeeper).to receive(:new).with(subject).and_return('kerplunk')
+    expect(Nerve::Reporter.new_from_service(subject)).to eq('kerplunk')
+  end
+  it 'actually constructs an instance of a specific backend' do
+    expect(Nerve::Reporter.new_from_service(subject).is_a?(Nerve::Reporter::Zookeeper)).to eql(true)
+  end
+  it 'the reporter backend inherits from the base class' do
+    expect(Nerve::Reporter.new_from_service(subject).is_a?(Nerve::Reporter::Base)).to eql(true)
+  end
+  it 'throws ArgumentError if you ask for a reporter type which does not exist' do
+    subject['reporter_type'] = 'does_not_exist'
+    expect { Nerve::Reporter.new_from_service(subject) }.to raise_error(ArgumentError)
+  end
+end
+
+class Nerve::Reporter::Test < Nerve::Reporter::Base
+end
+
+describe Nerve::Reporter::Test do
+  let(:subject) {Nerve::Reporter::Test.new({}) }
+  context 'parse_data method' do
+    it 'has parse data method that passes strings' do
+      expect(subject.send(:parse_data, 'foobar')).to eql('foobar')
+    end
+    it 'jsonifies anything that is not a string' do
+      thing_to_parse = double()
+      expect(thing_to_parse).to receive(:to_json).and_return('{"some":"json"}')
+      expect(subject.send(:parse_data, thing_to_parse)).to eql('{"some":"json"}')
+    end
+  end
+
+  context 'get_service_data method' do
+    it 'throws on missing arguments' do
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666}) }.to raise_error(ArgumentError)
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'instance_id' => 'foobar'}) }.to raise_error(ArgumentError)
+      expect { subject.get_service_data({'port' => 6666, 'instance_id' => 'foobar'}) }.to raise_error(ArgumentError)
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar'}) }.not_to raise_error
+    end
+    it 'takes weight if present and +ve integer' do
+      expect(subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => 3})['weight']).to eql(3)
+    end
+    it 'takes weight if present and 0' do
+      expect(subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => 0})['weight']).to eql(0)
+    end
+    it 'skips weight if not present' do
+      expect(subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar'})['weight']).to eql(nil)
+    end
+    it 'skips weight if non integer' do
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => 'hello'})['weight'] }.to raise_error(ArgumentError)
+    end
+    it 'skips weight if a negative integer' do
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => -1})['weight'] }.to raise_error(ArgumentError)
+    end
+    it 'works if the weight is a string' do
+      expect(subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => '3'})['weight']).to eql(3)
+    end
+
+    it 'correctly passes haproxy_server_options' do
+      expect(subject.get_service_data({
+        'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar',
+        'haproxy_server_options' => 'backup'
+      })['haproxy_server_options']).to eql('backup')
+    end
+    it 'correctly passes labels' do
+      labels = {'az' => 'us-west-1', 'custom_key' => 'custom_value'}
+      expect(subject.get_service_data({
+        'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar',
+        'labels' => labels
+      })['labels']).to eql(labels)
+    end
+  end
+end
+

data/spec/lib/nerve/reporter_zookeeper_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+require 'nerve/reporter/zookeeper'
+
+describe Nerve::Reporter::Zookeeper do
+  let(:subject) { {
+      'zk_hosts' => ['zkhost1', 'zkhost2'],
+      'zk_path' => 'zk_path',
+      'instance_id' => 'instance_id',
+      'host' => 'host',
+      'port' => 'port'
+    }
+  }
+  it 'actually constructs an instance' do
+    expect(Nerve::Reporter::Zookeeper.new(subject).is_a?(Nerve::Reporter::Zookeeper)).to eql(true)
+  end
+
+  it 'deregisters service on exit' do
+    zk = double("zk")
+    allow(zk).to receive(:close!)
+    expect(zk).to receive(:mkdir_p) { "zk_path" }
+    expect(zk).to receive(:create) { "full_path" }
+    expect(zk).to receive(:delete).with("full_path", anything())
+
+    allow(ZK).to receive(:new).and_return(zk)
+
+    reporter = Nerve::Reporter::Zookeeper.new(subject)
+    reporter.start
+    reporter.report_up
+    reporter.stop
+  end
+end
+

data/spec/lib/nerve/service_watcher_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+require 'timeout'
+
+describe Nerve::ServiceWatcher do
+  describe 'initialize' do
+    let(:service) { build(:service) }
+
+    it 'can successfully initialize' do
+      Nerve::ServiceWatcher.new(service)
+    end
+
+    it 'requires minimum parameters' do
+      %w[name instance_id host port].each do |req|
+        service_without = service.dup
+        service_without.delete(req)
+
+        expect { Nerve::ServiceWatcher.new(service_without) }.to raise_error
+      end
+    end
+  end
+
+  describe 'check_and_report' do
+    let(:service_watcher) { Nerve::ServiceWatcher.new(build(:service)) }
+    let(:reporter) { service_watcher.instance_variable_get(:@reporter) }
+
+    it 'pings the reporter' do
+      expect(reporter).to receive(:ping?)
+      service_watcher.check_and_report
+    end
+
+    it 'reports the service as down when the checks fail' do
+      expect(service_watcher).to receive(:check?).and_return(false)
+      expect(reporter).to receive(:report_down)
+      service_watcher.check_and_report
+    end
+
+    it 'reports the service as up when the checks succeed' do
+      expect(service_watcher).to receive(:check?).and_return(true)
+      expect(reporter).to receive(:report_up)
+      service_watcher.check_and_report
+    end
+
+    it 'doesn\'t report if the status hasn\'t changed' do
+      expect(service_watcher).to receive(:check?).and_return(true)
+
+      expect(reporter).to receive(:report_up).once
+      expect(reporter).not_to receive(:report_down)
+      service_watcher.check_and_report
+    end
+  end
+
+  describe 'run' do
+    let(:check_interval) { 0 }
+    let(:service_watcher) { Nerve::ServiceWatcher.new(build(:service, :check_interval => check_interval)) }
+    let(:reporter) { service_watcher.instance_variable_get(:@reporter) }
+    before { $EXIT = false }
+
+    it 'starts the reporter' do
+      $EXIT = true
+      expect(reporter).to receive(:start)
+      service_watcher.run()
+    end
+
+    it 'calls check and report repeatedly' do
+      count = 0
+
+      # expect it to be called twice
+      expect(service_watcher).to receive(:check_and_report).twice do
+        # on the second call, set exit to true
+        $EXIT = true if count == 1
+        count += 1
+      end
+
+      service_watcher.run()
+    end
+
+    context 'when the check interval is long' do
+      let(:check_interval) { 10 }
+
+      it 'still exits quickly during nap time' do
+        expect(service_watcher).to receive(:check_and_report) do
+          $EXIT = true
+        end
+
+        expect{ Timeout::timeout(1) { service_watcher.run() } }.not_to raise_error
+      end
+    end
+  end
+end