nerve 0.5.4 → 0.6.0

data/.gitignore

@@ -19,3 +19,4 @@ tmp
 .\#*
 vendor
 .vagrant
+.ruby-version

data/.travis.yml

@@ -1,5 +1,8 @@
 language: ruby
 cache: bundler
+sudo: false
 rvm:
   - 1.9.3
-
+  - 2.0.0
+  - 2.1.6
+  - 2.2.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    nerve (0.5.4)
+    nerve (0.6.0)
       bunny (= 1.1.0)
       etcd (~> 0.2.3)
       json
@@ -34,7 +34,7 @@ GEM
     method_source (0.8.2)
     minitest (5.5.1)
     mixlib-log (1.6.0)
-    multi_json (1.10.1)
+    multi_json (1.11.2)
     pry (0.10.1)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -70,3 +70,6 @@ DEPENDENCIES
   pry
   rake
   rspec (~> 3.1.0)
+
+BUNDLED WITH
+   1.10.5

data/README.md

@@ -39,6 +39,7 @@ An example config file is available in `example/nerve.conf.json`.
 The config file is composed of two main sections:
 
 * `instance_id`: the name nerve will submit when registering services; makes debugging easier
+* `heartbeat_path`: a path to a file on disk to touch as nerve makes progress. This allows you to work around https://github.com/zk-ruby/zk/issues/50 by restarting a stuck nerve.
 * `services`: the hash (from service name to config) of the services nerve will be monitoring
 * `service_conf_dir`: path to a directory in which each json file will be interpreted as a service with the basename of the file minus the .json extension
 
@@ -53,6 +54,7 @@ The configuration contains the following options:
 * `reporter_type`: the mechanism used to report up/down information; depending on the reporter you choose, additional parameters may be required. Defaults to `zookeeper`
 * `check_interval`: the frequency with which service checks will be initiated; defaults to `500ms`
 * `checks`: a list of checks that nerve will perform; if all of the pass, the service will be registered; otherwise, it will be un-registered
+* `weight`: a positive integer weight value which can be used to affect the haproxy backend weighting in synapse.
 
 #### Zookeeper Reporter ####
 

data/example/nerve_services/etcd_service1.json

@@ -6,6 +6,7 @@
   "etcd_port": 4001,
   "etcd_path": "/nerve/services/your_http_service/services",
   "check_interval": 2,
+  "weight": 2,
   "checks": [
     {
       "type": "http",

data/example/nerve_services/zookeeper_service1.json

@@ -5,6 +5,7 @@
   "zk_hosts": ["localhost:2181"],
   "zk_path": "/nerve/services/your_http_service/services",
   "check_interval": 2,
+  "weight": 2,
   "checks": [
     {
       "type": "http",

data/lib/nerve/reporter/base.rb

@@ -19,10 +19,23 @@ class Nerve::Reporter
     def report_down
     end
 
-    def update_data(new_data='')
+    def ping?
     end
 
-    def ping?
+    def get_service_data(service)
+      %w{instance_id host port}.each do |required|
+        raise ArgumentError, "missing required argument #{required} for new service watcher" unless service[required]
+      end
+      d = {'host' => service['host'], 'port' => service['port'], 'name' => service['instance_id']}
+      return d unless service.has_key?('weight')
+
+      # Weight is optional, but it should be well formed if supplied
+      if service['weight'].to_i >= 0 and "#{service['weight']}".match /^\d+$/
+        d['weight'] = service['weight'].to_i
+      else
+        raise ArgumentError, "invalid 'weight' argument in service data: #{service.inspect}"
+      end
+      d
     end
 
     protected

data/lib/nerve/reporter/etcd.rb

@@ -4,14 +4,12 @@ require 'etcd'
 class Nerve::Reporter
   class Etcd < Base
     def initialize(service)
-      %w{etcd_host instance_id host port}.each do |required|
-        raise ArgumentError, "missing required argument #{required} for new service watcher" unless service[required]
-      end
+      raise ArgumentError, "missing required argument etcd_host for new service watcher" unless service['etcd_host']
       @host = service['etcd_host']
       @port = service['etcd_port'] || 4003
       path = service['etcd_path'] || '/'
       @path = path.split('/').push(service['instance_id']).join('/')
-      @data = parse_data({'host' => service['host'], 'port' => service['port'], 'name' => service['instance_id']})
+      @data = parse_data(get_service_data(service))
       @key = nil
       @ttl = (service['check_interval'] || 0.5) * 5
       @ttl = @ttl.ceil
@@ -36,13 +34,6 @@ class Nerve::Reporter
       etcd_delete
     end
 
-    def update_data(new_data='')
-      # nothing in nerve calls this, but implement it like the zookeeper
-      # reporter just for fun.
-      @data = parse_data(new_data)
-      etcd_save
-    end
-
     def ping?
       # we get a ping every check_interval.
       if @key

data/lib/nerve/reporter/zookeeper.rb

@@ -1,30 +1,64 @@
 require 'nerve/reporter/base'
+require 'thread'
 require 'zk'
 
+
 class Nerve::Reporter
   class Zookeeper < Base
+    @@zk_pool = {}
+    @@zk_pool_count = {}
+    @@zk_pool_lock = Mutex.new
+
     def initialize(service)
-      %w{zk_hosts zk_path instance_id host port}.each do |required|
+      %w{zk_hosts zk_path}.each do |required|
         raise ArgumentError, "missing required argument #{required} for new service watcher" unless service[required]
       end
-      @path = service['zk_hosts'].shuffle.join(',') + service['zk_path']
-      @data = parse_data({'host' => service['host'], 'port' => service['port'], 'name' => service['instance_id']})
+      # Since we pool we get one connection per zookeeper cluster
+      @zk_connection_string = service['zk_hosts'].sort.join(',')
+      @data = parse_data(get_service_data(service))
 
-      @key = "/#{service['instance_id']}_"
+      @zk_path = service['zk_path']
+      @key_prefix = @zk_path + "/#{service['instance_id']}_"
       @full_key = nil
     end
 
     def start()
-      log.info "nerve: waiting to connect to zookeeper at #{@path}"
-      @zk = ZK.new(@path)
-
-      log.info "nerve: successfully created zk connection to #{@path}"
+      log.info "nerve: waiting to connect to zookeeper to #{@zk_connection_string}"
+      # Ensure that all Zookeeper reporters re-use a single zookeeper
+      # connection to any given set of zk hosts.
+      @@zk_pool_lock.synchronize {
+        unless @@zk_pool.has_key?(@zk_connection_string)
+          log.info "nerve: creating pooled connection to #{@zk_connection_string}"
+          @@zk_pool[@zk_connection_string] = ZK.new(@zk_connection_string, :timeout => 5)
+          @@zk_pool_count[@zk_connection_string] = 1
+          log.info "nerve: successfully created zk connection to #{@zk_connection_string}"
+        else
+          @@zk_pool_count[@zk_connection_string] += 1
+          log.info "nerve: re-using existing zookeeper connection to #{@zk_connection_string}"
+        end
+        @zk = @@zk_pool[@zk_connection_string]
+        log.info "nerve: retrieved zk connection to #{@zk_connection_string}"
+      }
     end
 
     def stop()
-      log.info "nerve: closing zk connection at #{@path}"
-      report_down
-      @zk.close!
+      log.info "nerve: removing zk node at #{@full_key}" if @full_key
+      begin
+        report_down
+      ensure
+        @@zk_pool_lock.synchronize {
+          @@zk_pool_count[@zk_connection_string] -= 1
+          # Last thread to use the connection closes it
+          if @@zk_pool_count[@zk_connection_string] == 0
+            log.info "nerve: closing zk connection to #{@zk_connection_string}"
+            begin
+              @zk.close!
+            ensure
+              @@zk_pool.delete(@zk_connection_string)
+            end
+          end
+        }
+      end
     end
 
     def report_up()
@@ -35,13 +69,8 @@ class Nerve::Reporter
       zk_delete
     end
 
-    def update_data(new_data='')
-      @data = parse_data(new_data)
-      zk_save
-    end
-
     def ping?
-      return @zk.ping?
+      return @zk.connected? && @zk.exists?(@full_key || '/')
     end
 
     private
@@ -54,7 +83,8 @@ class Nerve::Reporter
     end
 
     def zk_create
-      @full_key = @zk.create(@key, :data => @data, :mode => :ephemeral_sequential)
+      @zk.mkdir_p(@zk_path)
+      @full_key = @zk.create(@key_prefix, :data => @data, :mode => :ephemeral_sequential)
     end
 
     def zk_save

data/lib/nerve/service_watcher/http.rb

@@ -40,7 +40,7 @@ module Nerve
           log.debug "nerve: check #{@name} got response code #{code} with body \"#{body}\""
           return true
         else
-          log.error "nerve: check #{@name} got response code #{code} with body \"#{body}\""
+          log.warn "nerve: check #{@name} got response code #{code} with body \"#{body}\""
           return false
         end
       end

data/lib/nerve/service_watcher.rb

@@ -83,12 +83,15 @@ module Nerve
       raise e
     ensure
       log.info "nerve: ending service watch #{@name}"
-      $EXIT = true
       @reporter.stop
     end
 
     def check_and_report
-      @reporter.ping?
+      if !@reporter.ping?
+        # If the reporter can't ping, then we do not know the status
+        # and must force a new report.
+        @was_up = nil
+      end
 
       # what is the status of the service?
       is_up = check?

data/lib/nerve/version.rb

@@ -1,3 +1,3 @@
 module Nerve
-  VERSION = "0.5.4"
+  VERSION = "0.6.0"
 end

data/lib/nerve.rb

@@ -1,3 +1,4 @@
+require 'fileutils'
 require 'logger'
 require 'json'
 require 'timeout'
@@ -28,12 +29,9 @@ module Nerve
 
       @instance_id = opts['instance_id']
       @services = opts['services']
+      @heartbeat_path = opts['heartbeat_path']
       @watchers = {}
 
-      # Any exceptions in the watcher threads should wake the main thread so
-      # that we can fail fast.
-      Thread.abort_on_exception = true
-
       log.debug 'nerve: completed init'
     end
 
@@ -45,15 +43,41 @@ module Nerve
       end
 
       begin
-        sleep
-      rescue StandardError => e
+        loop do
+          # Check that watcher threads are still alive, auto-remediate if they
+          # are not. Sometimes zookeeper flakes out or connections are lost to
+          # remote datacenter zookeeper clusters, failing is not an option
+          relaunch = []
+          @watchers.each do |name, watcher_thread|
+            unless watcher_thread.alive?
+              relaunch << name
+            end
+          end
+
+          relaunch.each do |name|
+            begin
+              log.warn "nerve: watcher #{name} not alive; reaping and relaunching"
+              reap_watcher(name)
+            rescue => e
+              log.warn "nerve: could not reap #{name}, got #{e.inspect}"
+            end
+            launch_watcher(name, @services[name])
+          end
+
+          unless @heartbeat_path.nil?
+            FileUtils.touch(@heartbeat_path)
+          end
+
+          sleep 10
+        end
+      rescue => e
         log.error "nerve: encountered unexpected exception #{e.inspect} in main thread"
         raise e
       ensure
         $EXIT = true
         log.warn 'nerve: reaping all watchers'
         @watchers.each do |name, watcher_thread|
-          reap_watcher(name)
+          reap_watcher(name) rescue "nerve: watcher #{name} could not be immediately reaped; skippping"
         end
       end
 

data/nerve.gemspec

@@ -8,9 +8,13 @@ Gem::Specification.new do |gem|
   gem.version       = Nerve::VERSION
   gem.authors       = ["Martin Rhoads", "Igor Serebryany", "Pierre Carrier"]
   gem.email         = ["martin.rhoads@airbnb.com", "igor.serebryany@airbnb.com"]
-  gem.description   = %q{description}
-  gem.summary       = %q{summary}
-  gem.homepage      = ""
+  gem.description   = "Nerve is a service registration daemon. It performs health "\
+                      "checks on your service and then publishes success or failure "\
+                      "into one of several registries (currently, zookeeper or etcd). "\
+                      "Nerve is half or SmartStack, and is designed to be operated "\
+                      "along with Synapse to provide a full service discovery framework"
+  gem.summary       = %q{A service registration daemon}
+  gem.homepage      = "https://github.com/airbnb/nerve"
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/spec/example_services_spec.rb

@@ -1,7 +1,12 @@
 require 'json'
 require 'nerve/reporter'
+require 'nerve/reporter/base'
 require 'nerve/service_watcher'
 
+class Nerve::Reporter::Base
+  attr_reader :data
+end
+
 describe "example services are valid" do
   Dir.foreach("#{File.dirname(__FILE__)}/../example/nerve_services") do |item|
     next if item == '.' or item == '..'
@@ -15,6 +20,9 @@ describe "example services are valid" do
         expect { reporter = Nerve::Reporter.new_from_service(service_data) }.to_not raise_error()
         expect(reporter.is_a?(Nerve::Reporter::Base)).to eql(true)
       end
+      it 'saves the weight data' do
+        expect(JSON.parse(Nerve::Reporter.new_from_service(service_data).data)['weight']).to eql(2)
+      end
     end
 
     context "when #{item} can be initialized as a valid service watcher" do

data/spec/lib/nerve/reporter_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'nerve/reporter/zookeeper'
 
 describe Nerve::Reporter do
   let(:subject) { {
@@ -30,13 +31,42 @@ end
 
 describe Nerve::Reporter::Test do
   let(:subject) {Nerve::Reporter::Test.new({}) }
-  it 'has parse data method that passes strings' do
-    expect(subject.send(:parse_data, 'foobar')).to eql('foobar')
+  context 'parse_data method' do
+    it 'has parse data method that passes strings' do
+      expect(subject.send(:parse_data, 'foobar')).to eql('foobar')
+    end
+    it 'jsonifies anything that is not a string' do
+      thing_to_parse = double()
+      expect(thing_to_parse).to receive(:to_json).and_return('{"some":"json"}')
+      expect(subject.send(:parse_data, thing_to_parse)).to eql('{"some":"json"}')
+    end
   end
-  it 'jsonifies anything that is not a string' do
-    thing_to_parse = double()
-    expect(thing_to_parse).to receive(:to_json).and_return('{"some":"json"}')
-    expect(subject.send(:parse_data, thing_to_parse)).to eql('{"some":"json"}')
+
+  context 'get_service_data method' do
+    it 'throws on missing arguments' do
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666}) }.to raise_error(ArgumentError)
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'instance_id' => 'foobar'}) }.to raise_error(ArgumentError)
+      expect { subject.get_service_data({'port' => 6666, 'instance_id' => 'foobar'}) }.to raise_error(ArgumentError)
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar'}) }.not_to raise_error
+    end
+    it 'takes weight if present and +ve integer' do
+      expect(subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => 3})['weight']).to eql(3)
+    end
+    it 'takes weight if present and 0' do
+      expect(subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => 0})['weight']).to eql(0)
+    end
+    it 'skips weight if not present' do
+      expect(subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar'})['weight']).to eql(nil)
+    end
+    it 'skips weight if non integer' do
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => 'hello'})['weight'] }.to raise_error(ArgumentError)
+    end
+    it 'skips weight if a negative integer' do
+      expect { subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => -1})['weight'] }.to raise_error(ArgumentError)
+    end
+    it 'works if the weight is a string' do
+      expect(subject.get_service_data({'host' => '127.0.0.1', 'port' => 6666, 'instance_id' => 'foobar', 'weight' => '3'})['weight']).to eql(3)
+    end
   end
 end
 

data/spec/lib/nerve/reporter_zookeeper_spec.rb

@@ -17,6 +17,7 @@ describe Nerve::Reporter::Zookeeper do
   it 'deregisters service on exit' do
     zk = double("zk")
     allow(zk).to receive(:close!)
+    expect(zk).to receive(:mkdir_p) { "zk_path" }
     expect(zk).to receive(:create) { "full_path" }
     expect(zk).to receive(:delete).with("full_path", anything())
 

data/spec/spec_helper.rb

@@ -12,7 +12,8 @@ FactoryGirl.find_definitions
 RSpec.configure do |config|
   config.run_all_when_everything_filtered = true
   config.filter_run :focus
-  config.include Config
+  config.include RbConfig
+  config.color = true
 
   # verify every double we can think of
   config.mock_with :rspec do |mocks|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nerve
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-19 00:00:00.000000000 Z
+date: 2015-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -141,7 +141,10 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: description
+description: Nerve is a service registration daemon. It performs health checks on
+  your service and then publishes success or failure into one of several registries
+  (currently, zookeeper or etcd). Nerve is half or SmartStack, and is designed to
+  be operated along with Synapse to provide a full service discovery framework
 email:
 - martin.rhoads@airbnb.com
 - igor.serebryany@airbnb.com
@@ -189,7 +192,7 @@ files:
 - spec/lib/nerve/reporter_zookeeper_spec.rb
 - spec/lib/nerve/service_watcher_spec.rb
 - spec/spec_helper.rb
-homepage: ''
+homepage: https://github.com/airbnb/nerve
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -212,7 +215,7 @@ rubyforge_project:
 rubygems_version: 1.8.23.2
 signing_key: 
 specification_version: 3
-summary: summary
+summary: A service registration daemon
 test_files:
 - spec/.gitkeep
 - spec/example_services_spec.rb