nerve 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 09be8c301fe459dcab4e5aa469f863ebb27c9f51
-  data.tar.gz: f458f7c7d2cbbd3e82f029d3a3488bcd5eff9088
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZDU0ZGNiMzk5OGRiZGU4ZjQwYTFmMmZiNzA5NGFiMWU2ODIzOGY2MQ==
+  data.tar.gz: !binary |-
+    NzJkZTAwMmIyOTQ0MDEyMGFiYTZlOTJjMGM3YTUwMmJmYTVmYjNmNg==
 SHA512:
-  metadata.gz: bf15ca3c591c8bac19750d9c3e1ba96ef7af6096492dbd69f76c971563a22fb7948fa1322d11664b819e185f6d51b1e279ba54b08bba062433c6fb06b7b3be5b
-  data.tar.gz: b4b4b89e7eb356611b11aea64f9df687af5c1dc657864d791d799f7b80b2c7e63c16b7fccdc4e502593e03b7f42b6f6f425f2c67dddacaaa99bb0b4778ffd7cf
+  metadata.gz: !binary |-
+    ZjczNGM1MzEzOTFkNmJhOTdlYmI0MjE4YWE2ODQ4NWViNDY4MGI5Y2FkZDM3
+    NTA4MzQxNDUzZGE1MWIyODc3Y2MyOGQyYzgzZTE2ZGNjNGQ3MmExZDVmMTU0
+    ZmFjZTc1NzBjYjA2OWEyMjgwMDIxN2FiMTFjOWE0YThiYzFlYzc=
+  data.tar.gz: !binary |-
+    YTYyYzE3NDVmOTk4NGQzYTQ5MThlZWExOTdjNDY0OGMyMjU1MjhhYWRjNWE1
+    ZGZlOTgzNjcxZWUyYTIwNjVhMzNjNzY4NTg2ZTFjYzc0NzM1NDliOGIzNmEx
+    ZTY0MTkzZGE4NmNmMjdiYTRiN2RlMDQ4NDJhYWQ4OWNjYWNjZGI=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    nerve (0.7.0)
+    nerve (0.8.0)
       bunny (= 1.1.0)
       etcd (~> 0.2.3)
       json

data/bin/nerve

@@ -4,74 +4,13 @@ require 'yaml'
 require 'optparse'
 
 require 'nerve'
+require 'nerve/configuration_manager'
 
-options={}
-
-# set command line options
-optparse = OptionParser.new do |opts|
-  opts.banner =<<EOB
-Welcome to nerve
-
-Usage: nerve --config /path/to/nerve/config
-EOB
-
-  options[:config] = ENV['NERVE_CONFIG']
-  opts.on('-c config','--config config', String, 'path to nerve config') do |key,value|
-    options[:config] = key
-  end
-
-  options[:instance_id] = ENV['NERVE_INSTANCE_ID']
-  opts.on('-i instance_id','--instance_id instance_id', String,
-    'reported as `name` to ZK; overrides instance id from config file') do |key,value|
-    options[:instance_id] = key
-  end
-
-  opts.on('-h', '--help', 'Display this screen') do
-    puts opts
-    exit
-  end
-
-end
-
-
-# parse command line arguments
-optparse.parse!
-
-def parseconfig(filename)
-  # parse synapse config file
-  begin
-    c = YAML::parse(File.read(filename))
-  rescue Errno::ENOENT => e
-    raise ArgumentError, "config file does not exist:\n#{e.inspect}"
-  rescue Errno::EACCES => e
-    raise ArgumentError, "could not open config file:\n#{e.inspect}"
-  rescue YAML::ParseError => e
-    raise "config file #{filename} is not yaml:\n#{e.inspect}"
-  end
-  return c.to_ruby
-end
-
-config = parseconfig(options[:config])
-config['services'] ||= {}
-
-if config.has_key?('service_conf_dir')
-  cdir = File.expand_path(config['service_conf_dir'])
-  if ! Dir.exists?(cdir) then
-    raise "service conf dir does not exist:#{cdir}"
-  end
-  cfiles = Dir.glob(File.join(cdir, '*.{yaml,json}'))
-  cfiles.each { |x| config['services'][File.basename(x[/(.*)\.(yaml|json)$/, 1])] = parseconfig(x) }
-end
-
-if options[:instance_id] && !options[:instance_id].empty?
-  config['instance_id'] = options[:instance_id]
-end
-
-# create nerve object
-s = Nerve::Nerve.new config
-
-# start nerve
-s.run
+# Parse command line options and parse configuration
+config_manager = Nerve::ConfigurationManager.new()
+config_manager.parse_options!
 
+nerve_application = Nerve::Nerve.new(config_manager)
+nerve_application.run
 
 puts "exiting nerve"

data/lib/nerve/configuration_manager.rb

@@ -0,0 +1,90 @@
+require 'yaml'
+require 'optparse'
+
+module Nerve
+  class ConfigurationManager
+    attr_reader :options
+    attr_reader :config
+
+    def parse_options_from_argv!
+      options = {}
+      # set command line options
+      optparse = OptionParser.new do |opts|
+        opts.banner =<<EOB
+Welcome to nerve
+
+Usage: nerve --config /path/to/nerve/config
+EOB
+
+        options[:config] = ENV['NERVE_CONFIG']
+        opts.on('-c config','--config config', String, 'path to nerve config') do |key,value|
+          options[:config] = key
+        end
+
+        options[:instance_id] = ENV['NERVE_INSTANCE_ID']
+        opts.on('-i instance_id','--instance_id instance_id', String,
+          'reported as `name` to ZK; overrides instance id from config file') do |key,value|
+          options[:instance_id] = key
+        end
+
+        options[:check_config] = ENV['NERVE_CHECK_CONFIG']
+        opts.on('-k', '--check-config',
+                'Validate the nerve config ONLY and exit 0 if valid (non zero otherwise)') do |_|
+          options[:check_config] = true
+        end
+
+        opts.on('-h', '--help', 'Display this screen') do
+          puts opts
+          exit
+        end
+      end
+      optparse.parse!
+      options
+    end
+
+    def parse_options!
+     @options = parse_options_from_argv!
+    end
+
+    def generate_nerve_config(options)
+      config = parse_config_file(options[:config])
+      config['services'] ||= {}
+
+      if config.has_key?('service_conf_dir')
+        cdir = File.expand_path(config['service_conf_dir'])
+        if ! Dir.exists?(cdir) then
+          raise "service conf dir does not exist:#{cdir}"
+        end
+        cfiles = Dir.glob(File.join(cdir, '*.{yaml,json}'))
+        cfiles.each { |x| config['services'][File.basename(x[/(.*)\.(yaml|json)$/, 1])] = parse_config_file(x) }
+      end
+
+      if options[:instance_id] && !options[:instance_id].empty?
+        config['instance_id'] = options[:instance_id]
+      end
+
+      config
+    end
+
+    def parse_config_file(filename)
+      # parse nerve config file
+      begin
+        c = YAML::parse(File.read(filename))
+      rescue Errno::ENOENT => e
+        raise ArgumentError, "config file does not exist:\n#{e.inspect}"
+      rescue Errno::EACCES => e
+        raise ArgumentError, "could not open config file:\n#{e.inspect}"
+      rescue YAML::SyntaxError => e
+        raise "config file #{filename} is not proper yaml:\n#{e.inspect}"
+      end
+      return c.to_ruby
+    end
+
+    def reload!
+      raise "You must parse command line options before reloading config" if @options.nil?
+      @config = generate_nerve_config(@options)
+    end
+  end
+end
+
+

data/lib/nerve/service_watcher.rb

@@ -7,6 +7,8 @@ module Nerve
     include Utils
     include Logging
 
+    attr_reader :was_up
+
     def initialize(service={})
       log.debug "nerve: creating service watcher object"
 
@@ -17,7 +19,7 @@ module Nerve
 
       @name = service['name']
 
-      # configure the reporter, which we use for talking to zookeeper
+      # configure the reporter, which we use for reporting status to the registry 
       @reporter = Reporter.new_from_service(service)
 
       # instantiate the checks for this service
@@ -57,32 +59,57 @@ module Nerve
       # force an initial report on startup
       @was_up = nil
 
+      # when this watcher is started it will store the
+      # thread here
+      @run_thread = nil
+      @should_finish = false
+
       log.debug "nerve: created service watcher for #{@name} with #{@service_checks.size} checks"
     end
 
+    def start()
+      unless @run_thread
+        @run_thread = Thread.new { self.run() }
+      else
+        log.error "nerve: tried to double start a watcher"
+      end
+    end
+
+    def stop()
+      log.info "nerve: stopping service watch #{@name}"
+      @should_finish = true
+      return true if @run_thread.nil?
+
+      unclean_shutdown = @run_thread.join(10).nil?
+      if unclean_shutdown
+        log.error "nerve: unclean shutdown of #{@name}, killing thread"
+        Thread.kill(@run_thread)
+      end
+      @run_thread = nil
+      !unclean_shutdown
+    end
+
+    def alive?()
+      !@run_thread.nil? && @run_thread.alive?
+    end
+
     def run()
       log.info "nerve: starting service watch #{@name}"
-
       @reporter.start()
 
-      until $EXIT
+      until watcher_should_exit?
         check_and_report
 
         # wait to run more checks but make sure to exit if $EXIT
         # we avoid sleeping for the entire check interval at once
         # so that nerve can exit promptly if required
-        nap_time = @check_interval
-        while nap_time > 0
-          break if $EXIT
-          sleep [nap_time, 1].min
-          nap_time -= 1
-        end
+        responsive_sleep (@check_interval) { watcher_should_exit? }
       end
     rescue StandardError => e
       log.error "nerve: error in service watcher #{@name}: #{e.inspect}"
       raise e
     ensure
-      log.info "nerve: ending service watch #{@name}"
+      log.info "nerve: stopping reporter for #{@name}"
       @reporter.stop
     end
 
@@ -115,5 +142,11 @@ module Nerve
       end
       return true
     end
+
+    private
+    def watcher_should_exit?
+      $EXIT || @should_finish
+    end
+
   end
 end

data/lib/nerve/utils.rb

@@ -4,5 +4,14 @@ module Nerve
       res = `#{command}`.chomp
       raise "command '#{command}' failed to run:\n#{res}" unless $?.success?
     end
+
+    def responsive_sleep(seconds, tick=1, &should_exit)
+      nap_time = seconds
+      while nap_time > 0
+        break if (should_exit && should_exit.call)
+        sleep [nap_time, tick].min
+        nap_time -= tick
+      end
+    end
   end
 end

data/lib/nerve/version.rb

@@ -1,3 +1,3 @@
 module Nerve
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/lib/nerve.rb

@@ -12,44 +12,125 @@ require 'nerve/service_watcher'
 
 module Nerve
   class Nerve
-
     include Logging
+    include Utils
+
+    MAIN_LOOP_SLEEP_S = 10.freeze
+    LAUNCH_WAIT_FOR_REPORT_S = 30.freeze
 
-    def initialize(opts={})
-      log.info 'nerve: starting up!'
+    def initialize(config_manager)
+      log.info 'nerve: setting up!'
+      @config_manager = config_manager
 
       # set global variable for exit signal
       $EXIT = false
 
-      # required options
-      log.debug 'nerve: checking for required inputs'
-      %w{instance_id services}.each do |required|
-        raise ArgumentError, "you need to specify required argument #{required}" unless opts[required]
-      end
-
-      @instance_id = opts['instance_id']
-      @services = opts['services']
-      @heartbeat_path = opts['heartbeat_path']
+      # State of currently running watchers according to Nerve
       @watchers = {}
+      @watcher_versions = {}
+
+      # instance_id, heartbeat_path, and watchers_desired are populated by
+      # load_config! in the main loop from the configuration source
+      @instance_id = nil
+      @heartbeat_path = nil
+      @watchers_desired = {}
+
+      # Flag to indicate a config reload is required by the main loop
+      # This decoupling is required for gracefully reloading config on SIGHUP
+      # as one should do as little as possible in a signal handler
+      @config_to_load = true
+
+      Signal.trap("HUP") do
+        @config_to_load = true
+      end
 
       log.debug 'nerve: completed init'
     end
 
-    def run
-      log.info 'nerve: starting run'
+    def load_config!
+      log.info 'nerve: loading config'
+      @config_to_load = false
+      @config_manager.reload!
+      config = @config_manager.config
 
-      @services.each do |name, config|
-        launch_watcher(name, config)
+      # required options
+      log.debug 'nerve: checking for required inputs'
+      %w{instance_id services}.each do |required|
+        raise ArgumentError, "you need to specify required argument #{required}" unless config[required]
       end
+      @instance_id = config['instance_id']
+      @watchers_desired = config['services']
+      @heartbeat_path = config['heartbeat_path']
+    end
 
+    def run
+      log.info 'nerve: starting main run loop'
       begin
-        loop do
-          # Check that watcher threads are still alive, auto-remediate if they
+        until $EXIT
+          # Check if configuration needs to be reloaded and reconcile any new
+          # configuration of watchers with old configuration
+          if @config_to_load
+            load_config!
+
+            # Reap undesired service watchers
+            services_to_reap = @watchers.select{ |name, _|
+              !@watchers_desired.has_key?(name)
+            }.keys()
+
+            unless services_to_reap.empty?
+              log.info "nerve: reaping old watchers: #{services_to_reap}"
+              services_to_reap.each do |name|
+                  reap_watcher(name)
+              end
+            end
+
+            # Start new desired service watchers
+            services_to_launch = @watchers_desired.select{ |name, _|
+              !@watchers.has_key?(name)
+            }.keys()
+
+            unless services_to_launch.empty?
+              log.info "nerve: launching new watchers: #{services_to_launch}"
+              services_to_launch.each do |name|
+                launch_watcher(name, @watchers_desired[name])
+              end
+            end
+
+            # Detect and update existing service watchers which are in both
+            # the currently running state and the desired (config) watcher
+            # state but have different configurations
+            services_to_update = @watchers.select { |name, _|
+              @watchers_desired.has_key?(name) &&
+              merged_config(@watchers_desired[name], name).hash != @watcher_versions[name]
+            }.keys()
+
+            services_to_update.each do |name|
+              log.info "nerve: detected new config for #{name}"
+              # Keep the old watcher running until the replacement is launched
+              # This keeps the service registered while we change it over
+              # This also keeps connection pools active across diffs
+              temp_name = "#{name}_#{@watcher_versions[name]}"
+              @watchers[temp_name] = @watchers.delete(name)
+              @watcher_versions[temp_name] = @watcher_versions.delete(name)
+              log.info "nerve: launching new watcher for #{name}"
+              launch_watcher(name, @watchers_desired[name], :wait => true)
+              log.info "nerve: reaping old watcher #{temp_name}"
+              reap_watcher(temp_name)
+            end
+          end
+
+          # If this was a configuration check, bail out now
+          if @config_manager.options[:check_config]
+            log.info 'nerve: configuration check succeeded, exiting immediately'
+            break
+          end
+
+          # Check that watchers are still alive, auto-remediate if they
           # are not. Sometimes zookeeper flakes out or connections are lost to
           # remote datacenter zookeeper clusters, failing is not an option
           relaunch = []
-          @watchers.each do |name, watcher_thread|
-            unless watcher_thread.alive?
+          @watchers.each do |name, watcher|
+            unless watcher.alive?
               relaunch << name
             end
           end
@@ -61,14 +142,13 @@ module Nerve
             rescue => e
               log.warn "nerve: could not reap #{name}, got #{e.inspect}"
             end
-            launch_watcher(name, @services[name])
+            launch_watcher(name, @watchers_desired[name])
           end
 
-          unless @heartbeat_path.nil?
-            FileUtils.touch(@heartbeat_path)
-          end
+          # Indicate we've made progress
+          heartbeat()
 
-          sleep 10
+          responsive_sleep(MAIN_LOOP_SLEEP_S) { @config_to_load || $EXIT }
         end
       rescue => e
         log.error "nerve: encountered unexpected exception #{e.inspect} in main thread"
@@ -76,8 +156,8 @@ module Nerve
       ensure
         $EXIT = true
         log.warn 'nerve: reaping all watchers'
-        @watchers.each do |name, watcher_thread|
-          reap_watcher(name) rescue "nerve: watcher #{name} could not be immediately reaped; skippping"
+        @watchers.each do |name, _|
+          reap_watcher(name)
         end
       end
 
@@ -86,15 +166,48 @@ module Nerve
       $EXIT = true
     end
 
-    def launch_watcher(name, config)
-      log.debug "nerve: launching service watcher #{name}"
-      watcher = ServiceWatcher.new(config.merge({'instance_id' => @instance_id, 'name' => name}))
-      @watchers[name] = Thread.new{watcher.run}
+    def heartbeat
+      unless @heartbeat_path.nil?
+        FileUtils.touch(@heartbeat_path)
+      end
+      log.debug 'nerve: heartbeat'
+    end
+
+    def merged_config(config, name)
+      # Get a deep copy so sub-hashes are properly handled
+      deep_copy = Marshal.load(Marshal.dump(config))
+      return deep_copy.merge({'instance_id' => @instance_id, 'name' => name})
+    end
+
+    def launch_watcher(name, config, opts = {})
+      wait = opts[:wait] || false
+
+      watcher_config = merged_config(config, name)
+      # The ServiceWatcher may mutate the configs, so record the version before
+      # passing the config to the ServiceWatcher
+      @watcher_versions[name] = watcher_config.hash
+
+      watcher = ServiceWatcher.new(watcher_config)
+      unless @config_manager.options[:check_config]
+        log.debug "nerve: launching service watcher #{name}"
+        watcher.start()
+        @watchers[name] = watcher
+        if wait
+          log.info "nerve: waiting for watcher thread #{name} to report"
+          responsive_sleep(LAUNCH_WAIT_FOR_REPORT_S) { !watcher.was_up.nil? || $EXIT }
+          log.info "nerve: watcher thread #{name} has reported!"
+        end
+      else
+        log.info "nerve: not launching #{name} due to --check-config option"
+      end
     end
 
     def reap_watcher(name)
-      watcher_thread = @watchers.delete(name)
-      watcher_thread.join()
+      watcher = @watchers.delete(name)
+      @watcher_versions.delete(name)
+      shutdown_status = watcher.stop()
+      log.info "nerve: stopped #{name}, clean shutdown? #{shutdown_status}"
+      shutdown_status
     end
   end
 end

data/nerve.gemspec

@@ -6,8 +6,8 @@ require 'nerve/version'
 Gem::Specification.new do |gem|
   gem.name          = "nerve"
   gem.version       = Nerve::VERSION
-  gem.authors       = ["Martin Rhoads", "Igor Serebryany", "Pierre Carrier"]
-  gem.email         = ["martin.rhoads@airbnb.com", "igor.serebryany@airbnb.com"]
+  gem.authors       = ["Martin Rhoads", "Igor Serebryany", "Pierre Carrier", "Joseph Lynch"]
+  gem.email         = ["martin.rhoads@airbnb.com", "igor.serebryany@airbnb.com", "jlynch@yelp.com"]
   gem.description   = "Nerve is a service registration daemon. It performs health "\
                       "checks on your service and then publishes success or failure "\
                       "into one of several registries (currently, zookeeper or etcd). "\

data/spec/configuration_manager_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+require 'nerve/configuration_manager'
+
+describe Nerve::ConfigurationManager do
+  describe 'parsing config' do
+    let(:config_manager) { Nerve::ConfigurationManager.new() }
+    let(:nerve_config) { "#{File.dirname(__FILE__)}/../example/nerve.conf.json" }
+    let(:nerve_instance_id) { 'testid' }
+
+    it 'parses valid options' do
+      allow(config_manager).to receive(:parse_options_from_argv!) { {
+        :config => nerve_config,
+        :instance_id => nerve_instance_id,
+        :check_config => false
+      } }
+
+      expect{config_manager.reload!}.to raise_error(RuntimeError)
+      expect(config_manager.parse_options!).to eql({
+        :config => nerve_config,
+        :instance_id => nerve_instance_id,
+        :check_config => false
+      })
+      expect{config_manager.reload!}.not_to raise_error
+      expect(config_manager.config.keys()).to include('instance_id', 'services')
+      expect(config_manager.config['services'].keys()).to contain_exactly(
+        'your_http_service', 'your_tcp_service', 'rabbitmq_service',
+        'etcd_service1', 'zookeeper_service1'
+      )
+    end
+  end
+end

data/spec/lib/nerve_spec.rb

@@ -0,0 +1,186 @@
+require 'spec_helper'
+require 'nerve/configuration_manager'
+require 'nerve/service_watcher'
+require 'nerve/reporter'
+require 'nerve/reporter/base'
+require 'nerve'
+
+def make_mock_service_watcher
+    mock_service_watcher = instance_double(Nerve::ServiceWatcher)
+    allow(mock_service_watcher).to receive(:start)
+    allow(mock_service_watcher).to receive(:stop)
+    allow(mock_service_watcher).to receive(:alive?).and_return(true)
+    allow(mock_service_watcher).to receive(:was_up).and_return(true)
+    mock_service_watcher
+end
+
+describe Nerve::Nerve do
+  let(:config_manager) { Nerve::ConfigurationManager.new() }
+  let(:mock_config_manager) { instance_double(Nerve::ConfigurationManager) }
+  let(:nerve_config) { "#{File.dirname(__FILE__)}/../../example/nerve.conf.json" }
+  let(:nerve_instance_id) { 'testid' }
+  let(:mock_service_watcher_one) { make_mock_service_watcher() }
+  let(:mock_service_watcher_two) { make_mock_service_watcher() }
+  let(:mock_reporter) { Nerve::Reporter::Base.new({}) }
+
+  describe 'check run' do
+    subject {
+      expect(config_manager).to receive(:parse_options_from_argv!).and_return({
+        :config => nerve_config,
+        :instance_id => nerve_instance_id,
+        :check_config => true
+      })
+      config_manager.parse_options!
+      Nerve::Nerve.new(config_manager)
+    }
+
+    it 'starts up and checks config' do
+      expect{subject.run}.not_to raise_error
+    end
+  end
+
+  describe 'full application run' do
+    before(:each) {
+      $EXIT = false
+
+      allow(Nerve::Reporter).to receive(:new_from_service) {
+        mock_reporter
+      }
+      allow(Nerve::ServiceWatcher).to receive(:new) { |config|
+        if config['name'] == 'service1'
+          mock_service_watcher_one
+        else
+          mock_service_watcher_two
+        end
+      }
+
+      allow(mock_config_manager).to receive(:reload!) { }
+      allow(mock_config_manager).to receive(:config) { {
+        'instance_id' => nerve_instance_id,
+        'services' => {
+          'service1' => {
+            'host' => 'localhost',
+            'port' => 1234
+          },
+          'service2' => {
+            'host' => 'localhost',
+            'port' => 1235
+          },
+        }
+      } }
+      allow(mock_config_manager).to receive(:options) { {
+        :config => 'noop',
+        :instance_id => nerve_instance_id,
+        :check_config => false
+      } }
+
+    }
+
+    it 'does a regular run and finishes' do
+      nerve = Nerve::Nerve.new(mock_config_manager)
+
+      expect(nerve).to receive(:heartbeat) {
+        $EXIT = true
+      }
+
+      expect{ nerve.run }.not_to raise_error
+    end
+
+    it 'relaunches dead watchers' do
+      nerve = Nerve::Nerve.new(mock_config_manager)
+
+      iterations = 2
+
+      # One service will fail an alive? call and need to be respawned
+      expect(nerve).to receive(:launch_watcher).twice.with('service1', anything).and_call_original
+      expect(nerve).to receive(:reap_watcher).twice.with('service1').and_call_original
+      expect(nerve).to receive(:launch_watcher).once.with('service2', anything).and_call_original
+      expect(nerve).to receive(:reap_watcher).once.with('service2').and_call_original
+
+      expect(nerve).to receive(:heartbeat).exactly(iterations + 1).times do
+        if iterations == 2
+          expect(mock_service_watcher_one).to receive(:alive?).and_return(false)
+          nerve.instance_variable_set(:@config_to_load, true)
+        elsif iterations == 1
+          expect(mock_service_watcher_one).to receive(:alive?).and_return(true)
+          nerve.instance_variable_set(:@config_to_load, true)
+        else
+          $EXIT = true
+        end
+        iterations -= 1
+      end
+
+      expect{ nerve.run }.not_to raise_error
+    end
+
+    it 'responds to changes in configuration' do
+      nerve = Nerve::Nerve.new(mock_config_manager)
+
+      iterations = 4
+      expect(nerve).to receive(:heartbeat).exactly(iterations + 1).times do
+        if iterations == 4
+          expect(nerve.instance_variable_get(:@watchers).keys).to contain_exactly('service1', 'service2')
+
+          # Remove service2 from the config
+          expect(mock_config_manager).to receive(:config).and_return({
+            'instance_id' => nerve_instance_id,
+            'services' => {
+              'service1' => {
+                'host' => 'localhost',
+                'port' => 1234
+              },
+            }
+          })
+          nerve.instance_variable_set(:@config_to_load, true)
+        elsif iterations == 3
+          expect(nerve.instance_variable_get(:@watchers).keys).to contain_exactly('service1')
+          expect(nerve.instance_variable_get(:@config_to_load)).to eq(false)
+
+          # Change the configuration of service1
+          expect(mock_config_manager).to receive(:config).and_return({
+            'instance_id' => nerve_instance_id,
+            'services' => {
+              'service1' => {
+                'host' => 'localhost',
+                'port' => 1236
+              },
+            }
+          })
+          nerve.instance_variable_set(:@config_to_load, true)
+        elsif iterations == 2
+          expect(nerve.instance_variable_get(:@watchers).keys).to contain_exactly('service1')
+          expect(nerve.instance_variable_get(:@watchers_desired).keys).to contain_exactly('service1')
+          expect(nerve.instance_variable_get(:@watchers_desired)['service1']['port']).to eq(1236)
+          expect(nerve.instance_variable_get(:@config_to_load)).to eq(false)
+
+          # Add another service
+          expect(mock_config_manager).to receive(:config) { {
+            'instance_id' => nerve_instance_id,
+            'services' => {
+              'service1' => {
+                'host' => 'localhost',
+                'port' => 1236
+              },
+              'service4' => {
+                'host' => 'localhost',
+                'port' => 1235
+              },
+            }
+          } }
+
+          nerve.instance_variable_set(:@config_to_load, true)
+        elsif iterations == 1
+          expect(nerve.instance_variable_get(:@watchers).keys).to contain_exactly('service1', 'service4')
+          nerve.instance_variable_set(:@config_to_load, true)
+        else
+          $EXIT = true
+        end
+        iterations -= 1
+      end
+
+      expect{ nerve.run }.not_to raise_error
+    end
+
+  end
+end
+

data/spec/spec_helper.rb

@@ -7,6 +7,7 @@
 require "#{File.dirname(__FILE__)}/../lib/nerve"
 
 require 'factory_girl'
+
 FactoryGirl.find_definitions
 
 RSpec.configure do |config|

metadata

@@ -1,43 +1,44 @@
 --- !ruby/object:Gem::Specification
 name: nerve
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Martin Rhoads
 - Igor Serebryany
 - Pierre Carrier
+- Joseph Lynch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-17 00:00:00.000000000 Z
+date: 2016-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: zk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.9.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.9.2
 - !ruby/object:Gem::Dependency
@@ -58,70 +59,70 @@ dependencies:
   name: etcd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.2.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: factory_girl
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Nerve is a service registration daemon. It performs health checks on
@@ -131,15 +132,16 @@ description: Nerve is a service registration daemon. It performs health checks o
 email:
 - martin.rhoads@airbnb.com
 - igor.serebryany@airbnb.com
+- jlynch@yelp.com
 executables:
 - nerve
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".mailmap"
-- ".nerve.rc"
-- ".travis.yml"
+- .gitignore
+- .mailmap
+- .nerve.rc
+- .travis.yml
 - CONTRIBUTING.md
 - Gemfile
 - Gemfile.lock
@@ -152,6 +154,7 @@ files:
 - example/nerve_services/etcd_service1.json
 - example/nerve_services/zookeeper_service1.json
 - lib/nerve.rb
+- lib/nerve/configuration_manager.rb
 - lib/nerve/log.rb
 - lib/nerve/reporter.rb
 - lib/nerve/reporter/base.rb
@@ -167,6 +170,7 @@ files:
 - lib/nerve/version.rb
 - nerve.gemspec
 - spec/.gitkeep
+- spec/configuration_manager_spec.rb
 - spec/example_services_spec.rb
 - spec/factories/check.rb
 - spec/factories/service.rb
@@ -174,6 +178,7 @@ files:
 - spec/lib/nerve/reporter_spec.rb
 - spec/lib/nerve/reporter_zookeeper_spec.rb
 - spec/lib/nerve/service_watcher_spec.rb
+- spec/lib/nerve_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/airbnb/nerve
 licenses: []
@@ -184,22 +189,23 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A service registration daemon
 test_files:
 - spec/.gitkeep
+- spec/configuration_manager_spec.rb
 - spec/example_services_spec.rb
 - spec/factories/check.rb
 - spec/factories/service.rb
@@ -207,4 +213,5 @@ test_files:
 - spec/lib/nerve/reporter_spec.rb
 - spec/lib/nerve/reporter_zookeeper_spec.rb
 - spec/lib/nerve/service_watcher_spec.rb
+- spec/lib/nerve_spec.rb
 - spec/spec_helper.rb