synapse 0.15.1 → 0.16.2

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NWU1NjVhM2M0NDBhNmU1Mjg0NzllYzUwNGRmNThjODAxMGQyODFmMw==
-  data.tar.gz: !binary |-
-    MTVjZWRhYjM0YjJkNWVhY2Q2ZTY2NWU4NjBmZGI0YmNhZWNjNjI1OQ==
+SHA1:
+  metadata.gz: 849f3da2b6429321568cbe08a89e21f19a6a3413
+  data.tar.gz: b0bf173b22a7a7a158c346b325e25e7df4dc0f29
 SHA512:
-  metadata.gz: !binary |-
-    YTkzMTAwNTBhNTE0NmUwMjg5YjQ4ZTNmNTA5ZTAxZDVlMzhlN2U0YmM5NmVi
-    M2QwMjliOGVjOGVhYzBkYjMwYzliM2Y0Y2Q4ZDJjMDM4ODRmOTJiYjA5OWRl
-    ODc0N2ZmNzg2ZWE4OWMxNTQ3M2M3ZGJlMTY5MmZkYmViN2ZiZWQ=
-  data.tar.gz: !binary |-
-    ZGIxNDA5ZDJjZWU2MjY0YjU4NmI2ODE0YjlkNmFlNmM4OTM4OTQyMWNhZDJl
-    ZDM5NGRmOWVmYTljMmE0YmU3MTc0NDE5NTZiMWFjYTI1ZjM2Y2NkYjJmZDFi
-    YmM0Njc1NDQ1ZmNkMzkwMDZkMWRlM2IzNzczZGZhODI2MDJkNjU=
+  metadata.gz: b2ef9bab0eb75e9eba82eb66c2214cfd6e3e68120019c7f84dc7ee4f1152d67bee1fdc8e68e3b143b346b1784d5ad45898e9d0a6f9e16150e6d9c5a47edaaa07
+  data.tar.gz: cec2427ff960706bff57e38bb1bef434774698cc875bb9f32318bdba5c95c9035d04eface6e1d1502bcbdd0e5515281f486a256f9f8c1cdd3f832b30e1e30e91

data/README.md

@@ -230,7 +230,8 @@ It takes the following mandatory arguments:
 
 The watcher assumes that each node under `path` represents a service server.
 
-The watcher assumes that the data (if any) retrieved at znode `path` is a hash, where each key is named by a valid `config_generator` (e.g. `haproxy`) and the value is a hash that configs the generator.
+The watcher assumes that the data (if any) retrieved at znode `path` is a hash, where each key is named by a valid `config_generator` (e.g. `haproxy`) and the value is a hash that configs the generator.   Alternatively, if a `generator_config_path` argument is specified, the watcher will attempt to read generator config from that znode instead.
+If `generator_config_path` has the value `disabled`, then generator config will not be read from zookeeper at all.
 
 The following arguments are optional:
 
@@ -338,7 +339,7 @@ different addresses (example: service1 listen on 127.0.0.2:443 and service2
 listen on 127.0.0.3:443) allows /etc/hosts entries to point to services.
 * `bind_options`: optional: default value is an empty string, specify additional bind parameters, such as ssl accept-proxy, crt, ciphers etc.
 * `server_port_override`: **DEPRECATED**. Renamed [`backend_port_override`](#backend_port_override) and moved to the top level hash. This will be removed in future versions.
-* `server_options`: the haproxy options for each `server` line of the service in HAProxy config; it may be left out.
+* `server_options`: the haproxy options for each `server` line of the service in HAProxy config; it may be left out. This field supports some basic templating: you can add include `%{port}`, `%{host}`, or `%{name}` in this string, and those will be replaced with the appropriate values for the particular server being configured.
 * `frontend`: additional lines passed to the HAProxy config in the `frontend` stanza of this service
 * `backend`: additional lines passed to the HAProxy config in the `backend` stanza of this service
 * `backend_name`: The name of the generated HAProxy backend for this service
@@ -352,6 +353,7 @@ listen on 127.0.0.3:443) allows /etc/hosts entries to point to services.
   ordering is deterministic across HAProxy reloads.
 * `shared_frontend`: optional: haproxy configuration directives for a shared http frontend (see below)
 * `cookie_value_method`: optional: default value is `name`, it defines the way your backends receive a cookie value in http mode. If equal to `hash`, synapse hashes backend names on cookie value assignation of your discovered backends, useful when you want to use haproxy cookie feature but you do not want that your end users receive a Set-Cookie with your server name and ip readable in clear.
+* `use_nerve_weights`: optional: this option enables reading the weights from nerve and applying them to the haproxy configuration. By default this is disabled in the case where users apply weights using `server_options` or `haproxy_server_options`.  This option will also remove the weight parameter from `server_options` and `haproxy_server_options`
 
 <a name="haproxy"/>
 

data/bin/synapse

@@ -2,6 +2,7 @@
 
 require 'yaml'
 require 'optparse'
+require 'erb'
 
 require 'synapse'
 
@@ -32,13 +33,18 @@ optparse.parse!
 def parseconfig(filename)
   # parse synapse config file
   begin
-    c = YAML::parse(File.read(filename))
+    c = YAML::parse(ERB.new(File.read(filename)).result)
   rescue Errno::ENOENT => e
     raise ArgumentError, "config file does not exist:\n#{e.inspect}"
   rescue Errno::EACCES => e
     raise ArgumentError, "could not open config file:\n#{e.inspect}"
   rescue YAML::SyntaxError => e
     raise "config file #{filename} is not yaml:\n#{e.inspect}"
+  rescue SyntaxError => e
+    raise SyntaxError, "ERB syntax error in config file #{filename}:\n#{e.inspect}"
+  rescue
+    puts "failed to parse config file #{filename}"
+    raise
   end
   return c.to_ruby
 end

data/config/synapse.conf.json

@@ -25,7 +25,7 @@
           "http-check expect string OK"
         ]
       }
-      
+
     },
     "service2": {
       "default_servers": [
@@ -87,5 +87,16 @@
         "stats refresh 5s"
       ]
     }
+  },
+  "statsd": {
+    "host": "localhost",
+    "port": 8125,
+    "sample_rate": {
+      "synapse.watcher.ping.count": 0.1,
+      "synapse.watcher.zk.discovery": 0.7,
+      "synapse.watcher.zk.discovery.elapsed_time": 0.7,
+      "synapse.watcher.zk.get.elapsed_time": 0.5,
+      "synapse.watcher.zk.watch.elapsed_time": 0.7
+    }
   }
 }

data/lib/synapse.rb

@@ -3,6 +3,7 @@ require 'json'
 
 require 'synapse/version'
 require 'synapse/log'
+require 'synapse/statsd'
 require 'synapse/config_generator'
 require 'synapse/service_watcher'
 
@@ -11,8 +12,11 @@ module Synapse
   class Synapse
 
     include Logging
+    include StatsD
 
     def initialize(opts={})
+      StatsD.configure_statsd(opts["statsd"] || {})
+
       # create objects that need to be notified of service changes
       @config_generators = create_config_generators(opts)
       raise "no config generators supplied" if @config_generators.empty?
@@ -29,47 +33,76 @@ module Synapse
       Thread.abort_on_exception = true
 
       log.debug "synapse: completed init"
+    rescue Exception => e
+      statsd && statsd_increment('synapse.stop', ['stop_avenue:abort', 'stop_location:init', "exception_name:#{e.class.name}", "exception_message:#{e.message}"])
+      raise e
     end
 
     # start all the watchers and enable haproxy configuration
     def run
       log.info "synapse: starting..."
+      statsd_increment('synapse.start')
 
       # start all the watchers
-      @service_watchers.map { |watcher| watcher.start }
-
-      # main loop
-      loops = 0
-      loop do
-        @service_watchers.each do |w|
-          raise "synapse: service watcher #{w.name} failed ping!" unless w.ping?
+      statsd_time('synapse.watchers.start.time') do
+        @service_watchers.map do |watcher|
+          begin
+            watcher.start
+            statsd_increment("synapse.watcher.start", ['start_result:success', "watcher_name:#{watcher.name}"])
+          rescue Exception => e
+            statsd_increment("synapse.watcher.start", ['start_result:fail', "watcher_name:#{watcher.name}", "exception_name:#{e.class.name}", "exception_message:#{e.message}"])
+            raise e
+          end
         end
+      end
+
+      statsd_time('synapse.main_loop.elapsed_time') do
+        # main loop
+        loops = 0
+        loop do
+          @service_watchers.each do |w|
+            alive = w.ping?
+            statsd_increment('synapse.watcher.ping.count', ["watcher_name:#{w.name}", "ping_result:#{alive ? "success" : "failure"}"])
+            raise "synapse: service watcher #{w.name} failed ping!" unless alive
+          end
 
-        if @config_updated
-          @config_updated = false
+          if @config_updated
+            @config_updated = false
+            statsd_increment('synapse.config.update')
+            @config_generators.each do |config_generator|
+              log.info "synapse: configuring #{config_generator.name}"
+              config_generator.update_config(@service_watchers)
+            end
+          end
+
+          sleep 1
           @config_generators.each do |config_generator|
-            log.info "synapse: configuring #{config_generator.name}"
-            config_generator.update_config(@service_watchers)
+            config_generator.tick(@service_watchers)
           end
-        end
 
-        sleep 1
-        @config_generators.each do |config_generator|
-          config_generator.tick(@service_watchers)
+          loops += 1
+          log.debug "synapse: still running at #{Time.now}" if (loops % 60) == 0
         end
-
-        loops += 1
-        log.debug "synapse: still running at #{Time.now}" if (loops % 60) == 0
       end
 
     rescue StandardError => e
+      statsd_increment('synapse.stop', ['stop_avenue:abort', 'stop_location:main_loop', "exception_name:#{e.class.name}", "exception_message:#{e.message}"])
       log.error "synapse: encountered unexpected exception #{e.inspect} in main thread"
       raise e
     ensure
       log.warn "synapse: exiting; sending stop signal to all watchers"
 
       # stop all the watchers
-      @service_watchers.map(&:stop)
+      @service_watchers.map do |w|
+        begin
+          w.stop
+          statsd_increment("synapse.watcher.stop", ['stop_avenue:clean', 'stop_location:main_loop', "watcher_name:#{w.name}"])
+        rescue Exception => e
+          statsd_increment("synapse.watcher.stop", ['stop_avenue:exception', 'stop_location:main_loop', "watcher_name:#{w.name}", "exception_name:#{e.class.name}", "exception_message:#{e.message}"])
+          raise e
+        end
+      end
+      statsd_increment('synapse.stop', ['stop_avenue:clean', 'stop_location:main_loop'])
     end
 
     def reconfigure!
@@ -91,11 +124,18 @@ module Synapse
     end
 
     private
+
+    WAIVED_CONFIG_SECTIONS = [
+      'services',
+      'service_conf_dir',
+      'statsd',
+    ].freeze
+
     def create_config_generators(opts={})
       config_generators = []
       opts.each do |type, generator_opts|
         # Skip the "services" top level key
-        next if (type == 'services' || type == 'service_conf_dir')
+        next if WAIVED_CONFIG_SECTIONS.include? type
         config_generators << ConfigGenerator.create(type, generator_opts)
       end
 

data/lib/synapse/config_generator/file_output.rb

@@ -31,7 +31,7 @@ class Synapse::ConfigGenerator
 
     def update_config(watchers)
       watchers.each do |watcher|
-        next if watcher.config_for_generator[name]['disabled']
+        next if writer_disabled?(watcher)
 
         unless @watcher_revisions[watcher.name] == watcher.revision
           @watcher_revisions[watcher.name] = watcher.revision
@@ -68,12 +68,20 @@ class Synapse::ConfigGenerator
       # Cleanup old services that Synapse no longer manages
       FileUtils.cd(opts['output_directory']) do
         present_files = Dir.glob('*.json')
-        managed_watchers = current_watchers.reject {|watcher| watcher.config_for_generator[name]['disabled']}
+        managed_watchers = current_watchers.reject do |watcher|
+          writer_disabled?(watcher)
+        end
         managed_files = managed_watchers.collect {|watcher| "#{watcher.name}.json"}
         files_to_purge = present_files.select {|svc| not managed_files.include?(svc)}
         log.info "synapse: purging unknown service files #{files_to_purge}" if files_to_purge.length > 0
         FileUtils.rm(files_to_purge)
       end
     end
+
+    private
+
+    def writer_disabled?(watcher)
+      watcher.config_for_generator[name].nil? || watcher.config_for_generator[name]['disabled']
+    end
   end
 end

data/lib/synapse/config_generator/haproxy.rb

@@ -814,7 +814,6 @@ class Synapse::ConfigGenerator
       @opts['do_writes'] = true unless @opts.key?('do_writes')
       @opts['do_socket'] = true unless @opts.key?('do_socket')
       @opts['do_reloads'] = true unless @opts.key?('do_reloads')
-
       req_pairs = {
         'do_writes' => 'config_file_path',
         'do_socket' => 'socket_file_path',
@@ -1090,6 +1089,11 @@ class Synapse::ConfigGenerator
             log.info "synapse: restart required because haproxy_server_options changed for #{backend_name}"
             @restart_required = true
           end
+
+          if(@opts['use_nerve_weights'] && old_backend.fetch('weight', "") != backend.fetch('weight', ""))
+            log.info "synapse: restart required because weight changed for #{backend_name}"
+            @restart_required = true
+          end
         end
 
         backends[backend_name] = backend.merge('enabled' => true)
@@ -1099,8 +1103,8 @@ class Synapse::ConfigGenerator
         # does define an id, then we will write that out below, so that must be what
         # is in the id_map as well.
         @server_id_map[watcher.name][backend_name] = backend['haproxy_server_id'].to_i if backend['haproxy_server_id']
-        server_opts = (backend['haproxy_server_options'] || 'no match').split(' ')
-        @server_id_map[watcher.name][backend_name] = server_opts[server_opts.index('id') + 1].to_i if server_opts.include?('id')
+        server_opts = backend['haproxy_server_options'].split(' ') if backend['haproxy_server_options'].is_a? String
+        @server_id_map[watcher.name][backend_name] = server_opts[server_opts.index('id') + 1].to_i if server_opts && server_opts.include?('id')
         @id_server_map[watcher.name][@server_id_map[watcher.name][backend_name]] = backend_name
       end
 
@@ -1137,10 +1141,15 @@ class Synapse::ConfigGenerator
         config.map {|c| "\t#{c}"},
         keys.map {|backend_name|
           backend = backends[backend_name]
+          backend_template_vars = {
+            :host => backend['host'],
+            :port => backend['port'],
+            :name => backend_name,
+          }
           b = "\tserver #{backend_name} #{backend['host']}:#{backend['port']}"
 
           # Again, if the registry defines an id, we can't set it.
-          has_id = (backend['haproxy_server_options'] || 'no match').split(' ').include?('id')
+          has_id = backend['haproxy_server_options'].split(' ').include?('id') if backend['haproxy_server_options'].is_a? String
           if (!has_id && @server_id_map[watcher.name][backend_name])
             b = "#{b} id #{@server_id_map[watcher.name][backend_name]}"
           end
@@ -1153,13 +1162,37 @@ class Synapse::ConfigGenerator
               b = "#{b} cookie #{backend_name}"
             end
           end
-          b = "#{b} #{watcher_config['server_options']}" if watcher_config['server_options']
-          b = "#{b} #{backend['haproxy_server_options']}" if backend['haproxy_server_options']
+
+          if @opts['use_nerve_weights'] && backend['weight'] && (backend['weight'].is_a? Integer)
+            clean_server_options = remove_weight_option watcher_config['server_options']
+            clean_haproxy_server_options = remove_weight_option backend['haproxy_server_options']
+            if clean_server_options != watcher_config['server_options']
+                log.warn "synapse: weight is defined in both server_options and nerve. nerve weight will take precedence"
+            end
+            if clean_haproxy_server_options != backend['haproxy_server_options']
+                log.warn "synapse: weight is defined in both haproxy_server_options and nerve. nerve weight will take precedence"
+            end
+            b = "#{b} #{clean_server_options % backend_template_vars}" if clean_server_options
+            b = "#{b} #{clean_haproxy_server_options % backend_template_vars}" if clean_haproxy_server_options
+
+            weight = backend['weight'].to_i
+            b = "#{b} weight #{weight}".squeeze(" ")
+          else
+            b = "#{b} #{watcher_config['server_options'] % backend_template_vars}" if watcher_config['server_options'].is_a? String
+            b = "#{b} #{backend['haproxy_server_options'] % backend_template_vars}" if backend['haproxy_server_options'].is_a? String
+          end
           b = "#{b} disabled" unless backend['enabled']
           b }
       ]
     end
 
+    def remove_weight_option(server_options)
+      if server_options.is_a? String
+        server_options = server_options.sub /weight +[0-9]+/,''
+      end
+      server_options
+    end
+
     def find_next_id(watcher_name, backend_name)
       probe = nil
       if @server_id_map[watcher_name].size >= @max_server_id
@@ -1286,7 +1319,9 @@ class Synapse::ConfigGenerator
       if old_config == new_config
         return false
       else
-        File.open(opts['config_file_path'],'w') {|f| f.write(new_config)}
+        tmp_file_path = "#{opts['config_file_path']}.tmp"
+        File.write(tmp_file_path, new_config)
+        FileUtils.mv(tmp_file_path, opts['config_file_path'])
         return true
       end
     end

data/lib/synapse/service_watcher/base.rb

@@ -1,10 +1,12 @@
 require 'synapse/log'
+require 'synapse/statsd'
 require 'set'
 require 'hashdiff'
 
 class Synapse::ServiceWatcher
   class BaseWatcher
     include Synapse::Logging
+    include Synapse::StatsD
 
     LEADER_WARN_INTERVAL = 30
 

data/lib/synapse/service_watcher/dns.rb

@@ -59,10 +59,16 @@ class Synapse::ServiceWatcher
       end
     end
 
+    IP_REGEX = Regexp.union([Resolv::IPv4::Regex, Resolv::IPv6::Regex])
+
     def resolve_servers
       resolver.tap do |dns|
         resolution = discovery_servers.map do |server|
-          addresses = dns.getaddresses(server['host']).map(&:to_s)
+          if server['host'] =~ IP_REGEX
+            addresses = [server['host']]
+          else
+            addresses = dns.getaddresses(server['host']).map(&:to_s)
+          end
           [server, addresses.sort]
         end
 

data/lib/synapse/service_watcher/zookeeper.rb

@@ -30,12 +30,14 @@ class Synapse::ServiceWatcher
     end
 
     def start
-      @zk_hosts = @discovery['hosts'].sort.join(',')
+      zk_host_list = @discovery['hosts'].sort
+      @zk_cluster = host_list_to_cluster(zk_host_list)
+      @zk_hosts = zk_host_list.join(',')
 
       @watcher = nil
       @zk = nil
 
-      log.info "synapse: starting ZK watcher #{@name} @ hosts: #{@zk_hosts}, path: #{@discovery['path']}"
+      log.info "synapse: starting ZK watcher #{@name} @ cluster #{@zk_cluster} path: #{@discovery['path']}"
       zk_connect
     end
 
@@ -47,11 +49,22 @@ class Synapse::ServiceWatcher
     def ping?
       # @zk being nil implies no session *or* a lost session, do not remove
       # the check on @zk being truthy
-      @zk && @zk.connected?
+      # if the client is in any of the three states: associating, connecting, connected
+      # we consider it alive. this can avoid synapse restart on short network dis-connection
+      @zk && (@zk.associating? || @zk.connecting? || @zk.connected?)
     end
 
     private
 
+    def host_list_to_cluster(list)
+      first_host = list.sort.first
+      first_token = first_host.split('.').first
+      # extract cluster name by filtering name of first host
+      # remove domain extents and trailing numbers
+      last_non_number = first_token.rindex(/[^0-9]/)
+      last_non_number ? first_token[0..last_non_number] : first_host
+    end
+
     def validate_discovery_opts
       raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
         unless @discovery['method'] == 'zookeeper'
@@ -123,49 +136,78 @@ class Synapse::ServiceWatcher
 
     # find the current backends at the discovery path
     def discover
-      log.info "synapse: discovering backends for service #{@name}"
-
-      new_backends = []
-      @zk.children(@discovery['path'], :watch => true).each do |id|
-        begin
-          node = @zk.get("#{@discovery['path']}/#{id}")
-        rescue ZK::Exceptions::NoNode => e
-          # This can happen when the registry unregisters a service node between
-          # the call to @zk.children and @zk.get(path). ZK does not guarantee
-          # a read to ``get`` of a child returned by ``children`` will succeed
-          log.error("synapse: #{@discovery['path']}/#{id} disappeared before it could be read: #{e}")
-          next
+      statsd_increment('synapse.watcher.zk.discovery', ["zk_cluster:#{@zk_cluster}", "zk_path:#{@discovery['path']}", "service_name:#{@name}"])
+      statsd_time('synapse.watcher.zk.discovery.elapsed_time', ["zk_cluster:#{@zk_cluster}", "zk_path:#{@discovery['path']}", "service_name:#{@name}"]) do
+        log.info "synapse: discovering backends for service #{@name}"
+
+        new_backends = []
+        @zk.children(@discovery['path'], :watch => true).each do |id|
+          begin
+            node = statsd_time('synapse.watcher.zk.get.elapsed_time', ["zk_cluster:#{@zk_cluster}", "service_name:#{@name}"]) do
+              @zk.get("#{@discovery['path']}/#{id}")
+            end
+          rescue ZK::Exceptions::NoNode => e
+            # This can happen when the registry unregisters a service node between
+            # the call to @zk.children and @zk.get(path). ZK does not guarantee
+            # a read to ``get`` of a child returned by ``children`` will succeed
+            log.error("synapse: #{@discovery['path']}/#{id} disappeared before it could be read: #{e}")
+            next
+          end
+
+          begin
+            # TODO: Do less munging, or refactor out this processing
+            host, port, name, weight, haproxy_server_options, labels = deserialize_service_instance(node.first)
+          rescue StandardError => e
+            log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
+          else
+            # find the numberic id in the node name; used for leader elections if enabled
+            numeric_id = id.split('_').last
+            numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
+
+            log.debug "synapse: discovered backend #{name} at #{host}:#{port} for service #{@name}"
+            new_backends << {
+              'name' => name, 'host' => host, 'port' => port,
+              'id' => numeric_id, 'weight' => weight,
+              'haproxy_server_options' => haproxy_server_options,
+              'labels' => labels
+            }
+          end
         end
 
-        begin
-          # TODO: Do less munging, or refactor out this processing
-          host, port, name, weight, haproxy_server_options, labels = deserialize_service_instance(node.first)
-        rescue StandardError => e
-          log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
+        # support for a separate 'generator_config_path' key, for reading the
+        # generator config block, that may be different from the 'path' key where
+        # we discover service instances. if generator_config_path is present and
+        # the value is "disabled", then skip all zk-based discovery of the
+        # generator config (and use the values from the local config.json
+        # instead).
+        case @discovery.fetch('generator_config_path', nil)
+        when 'disabled'
+          discovery_key = nil
+        when nil
+          discovery_key = 'path'
         else
-          # find the numberic id in the node name; used for leader elections if enabled
-          numeric_id = id.split('_').last
-          numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
-
-          log.debug "synapse: discovered backend #{name} at #{host}:#{port} for service #{@name}"
-          new_backends << {
-            'name' => name, 'host' => host, 'port' => port,
-            'id' => numeric_id, 'weight' => weight,
-            'haproxy_server_options' => haproxy_server_options,
-            'labels' => labels
-          }
+          discovery_key = 'generator_config_path'
         end
-      end
 
-      node = @zk.get(@discovery['path'], :watch => true)
-      begin
-        new_config_for_generator = parse_service_config(node.first)
-      rescue StandardError => e
-        log.error "synapse: invalid config data in ZK node at #{@discovery['path']}: #{e}"
-        new_config_for_generator = {}
-      end
+        if discovery_key
+          begin
+            node = statsd_time('synapse.watcher.zk.get.elapsed_time', ["zk_cluster:#{@zk_cluster}", "service_name:#{@name}"]) do
+              @zk.get(@discovery[discovery_key], :watch => true)
+            end
+            new_config_for_generator = parse_service_config(node.first)
+          rescue ZK::Exceptions::NoNode => e
+            log.error "synapse: No ZK node for config data at #{@discovery[discovery_key]}: #{e}"
+            new_config_for_generator = {}
+          rescue StandardError => e
+            log.error "synapse: invalid config data in ZK node at #{@discovery[discovery_key]}: #{e}"
+            new_config_for_generator = {}
+          end
+        else
+          new_config_for_generator = {}
+        end
 
-      set_backends(new_backends, new_config_for_generator)
+        set_backends(new_backends, new_config_for_generator)
+      end
     end
 
     # sets up zookeeper callbacks if the data at the discovery path changes
@@ -173,12 +215,14 @@ class Synapse::ServiceWatcher
       return if @zk.nil?
       log.debug "synapse: setting watch at #{@discovery['path']}"
 
-      @watcher = @zk.register(@discovery['path'], &watcher_callback) unless @watcher
+      statsd_time('synapse.watcher.zk.watch.elapsed_time', ["zk_cluster:#{@zk_cluster}", "zk_path:#{@discovery['path']}", "service_name:#{@name}"]) do
+        @watcher = @zk.register(@discovery['path'], &watcher_callback) unless @watcher
 
-      # Verify that we actually set up the watcher.
-      unless @zk.exists?(@discovery['path'], :watch => true)
-        log.error "synapse: zookeeper watcher path #{@discovery['path']} does not exist!"
-        zk_cleanup
+        # Verify that we actually set up the watcher.
+        unless @zk.exists?(@discovery['path'], :watch => true)
+          log.error "synapse: zookeeper watcher path #{@discovery['path']} does not exist!"
+          zk_cleanup
+        end
       end
       log.debug "synapse: set watch at #{@discovery['path']}"
     end
@@ -221,37 +265,44 @@ class Synapse::ServiceWatcher
     end
 
     def zk_connect
-      log.info "synapse: zookeeper watcher connecting to ZK at #{@zk_hosts}"
-
-      # Ensure that all Zookeeper watcher re-use a single zookeeper
-      # connection to any given set of zk hosts.
-      @@zk_pool_lock.synchronize {
-        unless @@zk_pool.has_key?(@zk_hosts)
-          log.info "synapse: creating pooled connection to #{@zk_hosts}"
-          @@zk_pool[@zk_hosts] = ZK.new(@zk_hosts, :timeout => 5, :thread => :per_callback)
-          @@zk_pool_count[@zk_hosts] = 1
-          log.info "synapse: successfully created zk connection to #{@zk_hosts}"
-        else
-          @@zk_pool_count[@zk_hosts] += 1
-          log.info "synapse: re-using existing zookeeper connection to #{@zk_hosts}"
-        end
-      }
+      statsd_time('synapse.watcher.zk.connect.elapsed_time', ["zk_cluster:#{@zk_cluster}", "service_name:#{@name}"]) do
+        log.info "synapse: zookeeper watcher connecting to ZK at #{@zk_hosts}"
+
+        # Ensure that all Zookeeper watcher re-use a single zookeeper
+        # connection to any given set of zk hosts.
+        @@zk_pool_lock.synchronize {
+          unless @@zk_pool.has_key?(@zk_hosts)
+            log.info "synapse: creating pooled connection to #{@zk_hosts}"
+            @@zk_pool[@zk_hosts] = ZK.new(@zk_hosts, :timeout => 5, :thread => :per_callback)
+            @@zk_pool_count[@zk_hosts] = 1
+            log.info "synapse: successfully created zk connection to #{@zk_hosts}"
+            statsd_increment('synapse.watcher.zk.client.created', ["zk_cluster:#{@zk_cluster}", "service_name:#{@name}"])
+          else
+            @@zk_pool_count[@zk_hosts] += 1
+            log.info "synapse: re-using existing zookeeper connection to #{@zk_hosts}"
+            statsd_increment('synapse.watcher.zk.client.reused', ["zk_cluster:#{@zk_cluster}", "service_name:#{@name}"])
+          end
+        }
 
-      @zk = @@zk_pool[@zk_hosts]
-      log.info "synapse: retrieved zk connection to #{@zk_hosts}"
+        @zk = @@zk_pool[@zk_hosts]
+        log.info "synapse: retrieved zk connection to #{@zk_hosts}"
 
-      # handle session expiry -- by cleaning up zk, this will make `ping?`
-      # fail and so synapse will exit
-      @zk.on_expired_session do
-        log.warn "synapse: zookeeper watcher ZK session expired!"
-        zk_cleanup
-      end
+        # handle session expiry -- by cleaning up zk, this will make `ping?`
+        # fail and so synapse will exit
+        @zk.on_expired_session do
+          statsd_increment('synapse.watcher.zk.session.expired', ["zk_cluster:#{@zk_cluster}", "service_name:#{@name}"])
+          log.warn "synapse: zookeeper watcher ZK session expired!"
+          zk_cleanup
+        end
 
-      # the path must exist, otherwise watch callbacks will not work
-      create(@discovery['path'])
+        # the path must exist, otherwise watch callbacks will not work
+        statsd_time('synapse.watcher.zk.create_path.elapsed_time', ["zk_cluster:#{@zk_cluster}", "service_name:#{@name}"]) do
+          create(@discovery['path'])
+        end
 
-      # call the callback to bootstrap the process
-      watcher_callback.call
+        # call the callback to bootstrap the process
+        watcher_callback.call
+      end
     end
 
     # decode the data at a zookeeper endpoint