synapse-aurora 0.11.2

data/lib/synapse/service_watcher/zookeeper.rb

@@ -0,0 +1,153 @@
+require "synapse/service_watcher/base"
+
+require 'zk'
+
+module Synapse
+  class ZookeeperWatcher < BaseWatcher
+    NUMBERS_RE = /^\d+$/
+
+    def start
+      @zk_hosts = @discovery['hosts'].shuffle.join(',')
+
+      @watcher = nil
+      @zk = nil
+
+      log.info "synapse: starting ZK watcher #{@name} @ hosts: #{@zk_hosts}, path: #{@discovery['path']}"
+      zk_connect
+    end
+
+    def stop
+      log.warn "synapse: zookeeper watcher exiting"
+      zk_cleanup
+    end
+
+    def ping?
+      @zk && @zk.connected?
+    end
+
+    private
+
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'zookeeper'
+      raise ArgumentError, "missing or invalid zookeeper host for service #{@name}" \
+        unless @discovery['hosts']
+      raise ArgumentError, "invalid zookeeper path for service #{@name}" \
+        unless @discovery['path']
+    end
+
+    # helper method that ensures that the discovery path exists
+    def create(path)
+      log.debug "synapse: creating ZK path: #{path}"
+
+      # recurse if the parent node does not exist
+      create File.dirname(path) unless @zk.exists? File.dirname(path)
+      @zk.create(path, ignore: :node_exists)
+    end
+
+    # find the current backends at the discovery path; sets @backends
+    def discover
+      log.info "synapse: discovering backends for service #{@name}"
+
+      new_backends = []
+      @zk.children(@discovery['path'], :watch => true).each do |id|
+        node = @zk.get("#{@discovery['path']}/#{id}")
+
+        begin
+          host, port, name = deserialize_service_instance(node.first)
+        rescue StandardError => e
+          log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
+        else
+          server_port = @server_port_override ? @server_port_override : port
+
+          # find the numberic id in the node name; used for leader elections if enabled
+          numeric_id = id.split('_').last
+          numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
+
+          log.debug "synapse: discovered backend #{name} at #{host}:#{server_port} for service #{@name}"
+          new_backends << { 'name' => name, 'host' => host, 'port' => server_port, 'id' => numeric_id}
+        end
+      end
+
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name}; using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name}; using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        set_backends(new_backends)
+      end
+    end
+
+    # sets up zookeeper callbacks if the data at the discovery path changes
+    def watch
+      return if @zk.nil?
+
+      @watcher.unsubscribe unless @watcher.nil?
+      @watcher = @zk.register(@discovery['path'], &watcher_callback)
+
+      # Verify that we actually set up the watcher.
+      unless @zk.exists?(@discovery['path'], :watch => true)
+        log.error "synapse: zookeeper watcher path #{@discovery['path']} does not exist!"
+        raise RuntimeError.new('could not set a ZK watch on a node that should exist')
+      end
+    end
+
+    # handles the event that a watched path has changed in zookeeper
+    def watcher_callback
+      @callback ||= Proc.new do |event|
+        # Set new watcher
+        watch
+        # Rediscover
+        discover
+        # send a message to calling class to reconfigure
+        reconfigure!
+      end
+    end
+
+    def zk_cleanup
+      log.info "synapse: zookeeper watcher cleaning up"
+
+      @watcher.unsubscribe unless @watcher.nil?
+      @watcher = nil
+
+      @zk.close! unless @zk.nil?
+      @zk = nil
+
+      log.info "synapse: zookeeper watcher cleaned up successfully"
+    end
+
+    def zk_connect
+      log.info "synapse: zookeeper watcher connecting to ZK at #{@zk_hosts}"
+      @zk = ZK.new(@zk_hosts)
+
+      # handle session expiry -- by cleaning up zk, this will make `ping?`
+      # fail and so synapse will exit
+      @zk.on_expired_session do
+        log.warn "synapse: zookeeper watcher ZK session expired!"
+        zk_cleanup
+      end
+
+      # the path must exist, otherwise watch callbacks will not work
+      create(@discovery['path'])
+
+      # call the callback to bootstrap the process
+      watcher_callback.call
+    end
+
+    # decode the data at a zookeeper endpoint
+    def deserialize_service_instance(data)
+      log.debug "synapse: deserializing process data"
+      decoded = JSON.parse(data)
+
+      host = decoded['host'] || (raise ValueError, 'instance json data does not have host key')
+      port = decoded['port'] || (raise ValueError, 'instance json data does not have port key')
+      name = decoded['name'] || nil
+
+      return host, port, name
+    end
+  end
+end

data/lib/synapse/service_watcher/zookeeper_aurora.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+#
+# Variant of the Zookeeper service watcher that works with Apache Aurora
+# service announcements.
+#
+# Parameters:
+#   hosts: list of zookeeper hosts to query (List of Strings, required)
+#   path: "/path/to/serverset/in/zookeeper" (String, required)
+#   port_name: Named service endpoint (String, optional)
+#
+# If port_name is omitted, uses the default serviceEndpoint port.
+
+# zk node data looks like this:
+#
+# {
+#   "additionalEndpoints": {
+#     "aurora": {
+#       "host": "somehostname",
+#       "port": 31943
+#     },
+#     "http": {
+#       "host": "somehostname",
+#       "port": 31943
+#     },
+#     "otherport": {
+#       "host": "somehostname",
+#       "port": 31944
+#     }
+#   },
+#   "serviceEndpoint": {
+#     "host": "somehostname",
+#     "port": 31943
+#   },
+#   "shard": 0,
+#   "status": "ALIVE"
+# }
+#
+
+require 'synapse/service_watcher/zookeeper'
+
+module Synapse
+  # Watcher for Zookeeper announcements from Apache Aurora
+  class ZookeeperAuroraWatcher < Synapse::ZookeeperWatcher
+    def validate_discovery_opts
+      @discovery['method'] == 'zookeeper_aurora' ||
+        fail(ArgumentError,
+             "Invalid discovery method: #{@discovery['method']}")
+      @discovery['hosts'] ||
+        fail(ArgumentError,
+             "Missing or invalid zookeeper host for service #{@name}")
+      @discovery['path'] ||
+        fail(ArgumentError, "Invalid zookeeper path for service #{@name}")
+    end
+
+    def deserialize_service_instance(data)
+      log.debug 'Deserializing process data'
+      decoded = JSON.parse(data)
+
+      name = decoded['shard'].to_s ||
+        fail("Instance JSON data missing 'shard' key")
+
+      hostport = if @discovery['port_name']
+                   decoded['additionalEndpoints'][@discovery['port_name']] ||
+                     fail("Endpoint '#{@discovery['port_name']}' not found " \
+                          'in instance JSON data')
+                 else
+                   decoded['serviceEndpoint']
+                 end
+
+      host = hostport['host'] || fail("Instance JSON data missing 'host' key")
+      port = hostport['port'] || fail("Instance JSON data missing 'port' key")
+
+      [host, port, name]
+    end
+  end
+end

data/lib/synapse/service_watcher/zookeeper_dns.rb

@@ -0,0 +1,232 @@
+require 'synapse/service_watcher/base'
+require 'synapse/service_watcher/dns'
+require 'synapse/service_watcher/zookeeper'
+
+require 'thread'
+
+# Watcher for watching Zookeeper for entries containing DNS names that are
+# continuously resolved to IP Addresses.  The use case for this watcher is to
+# allow services that are addressed by DNS to be reconfigured via Zookeeper
+# instead of an update of the synapse config.
+#
+# The implementation builds on top of the existing DNS and Zookeeper watchers.
+# This watcher creates a thread to manage the lifecycle of the DNS and
+# Zookeeper watchers.  This thread also publishes messages on a queue to
+# indicate that DNS should be re-resolved (after the check interval) or that
+# the DNS watcher should be shut down.  The Zookeeper watcher waits for changes
+# in backends from zookeeper and publishes those changes on an internal queue
+# consumed by the DNS watcher.  The DNS watcher blocks on this queue waiting
+# for messages indicating that new servers are available, the check interval
+# has passed (triggering a re-resolve), or that the watcher should shut down.
+# The DNS watcher is responsible for the actual reconfiguring of backends.
+module Synapse
+  class ZookeeperDnsWatcher < BaseWatcher
+
+    # Valid messages that can be passed through the internal message queue
+    module Messages
+      class InvalidMessageError < RuntimeError; end
+
+      # Indicates new servers identified by DNS names to be resolved.  This is
+      # sent from Zookeeper on events that modify the ZK node. The payload is
+      # an array of hashes containing {'host', 'port', 'name'}
+      class NewServers < Struct.new(:servers); end
+
+      # Indicates that DNS should be re-resolved.  This is sent by the
+      # ZookeeperDnsWatcher thread every check_interval seconds to cause a
+      # refresh of the IP addresses.
+      class CheckInterval; end
+
+      # Indicates that the DNS watcher should shut down.  This is sent when
+      # stop is called.
+      class StopWatcher; end
+
+      # Saved instances of message types with contents that cannot vary.  This
+      # reduces object allocation.
+      STOP_WATCHER_MESSAGE = StopWatcher.new
+      CHECK_INTERVAL_MESSAGE = CheckInterval.new
+    end
+
+    class Dns < Synapse::DnsWatcher
+
+      # Overrides the discovery_servers method on the parent class
+      attr_accessor :discovery_servers
+
+      def initialize(opts={}, synapse, message_queue)
+        @message_queue = message_queue
+
+        super(opts, synapse)
+      end
+
+      def stop
+        @message_queue.push(Messages::STOP_WATCHER_MESSAGE)
+      end
+
+      def watch
+        last_resolution = nil
+        while true
+          # Blocks on message queue, the message will be a signal to stop
+          # watching, to check a new set of servers from ZK, or to re-resolve
+          # the DNS (triggered every check_interval seconds)
+          message = @message_queue.pop
+
+          log.debug "synapse: received message #{message.inspect}"
+
+          case message
+          when Messages::StopWatcher
+            break
+          when Messages::NewServers
+            self.discovery_servers = message.servers
+          when Messages::CheckInterval
+            # Proceed to re-resolve the DNS
+          else
+            raise Messages::InvalidMessageError,
+              "Received unrecognized message: #{message.inspect}"
+          end
+
+          # Empty servers means we haven't heard back from ZK yet or ZK is
+          # empty.  This should only occur if we don't get results from ZK
+          # within check_interval seconds or if ZK is empty.
+          if self.discovery_servers.nil? || self.discovery_servers.empty?
+            log.warn "synapse: no backends for service #{@name}"
+          else
+            # Resolve DNS names with the nameserver
+            current_resolution = resolve_servers
+            unless last_resolution == current_resolution
+              last_resolution = current_resolution
+              configure_backends(last_resolution)
+            end
+          end
+        end
+      end
+
+      private
+
+      # Validation is skipped as it has already occurred in the parent watcher
+      def validate_discovery_opts
+      end
+    end
+
+    class Zookeeper < Synapse::ZookeeperWatcher
+      def initialize(opts={}, synapse, message_queue)
+        super(opts, synapse)
+
+        @message_queue = message_queue
+      end
+
+      # Overrides reconfigure! to cause the new list of servers to be messaged
+      # to the DNS watcher rather than invoking a synapse reconfigure directly
+      def reconfigure!
+        # push the new backends onto the queue
+        @message_queue.push(Messages::NewServers.new(@backends))
+      end
+
+      private
+
+      # Validation is skipped as it has already occurred in the parent watcher
+      def validate_discovery_opts
+      end
+    end
+
+    def start
+      dns_discovery_opts = @discovery.select do |k,_|
+        k == 'nameserver'
+      end
+
+      zookeeper_discovery_opts = @discovery.select do |k,_|
+        k == 'hosts' || k == 'path'
+      end
+
+      @check_interval = @discovery['check_interval'] || 30.0
+
+      @message_queue = Queue.new
+
+      @dns = Dns.new(
+        mk_child_watcher_opts(dns_discovery_opts),
+        @synapse,
+        @message_queue
+      )
+
+      @zk = Zookeeper.new(
+        mk_child_watcher_opts(zookeeper_discovery_opts),
+        @synapse,
+        @message_queue
+      )
+
+      @zk.start
+      @dns.start
+
+      @watcher = Thread.new do
+        until @should_exit
+          # Trigger a DNS resolve every @check_interval seconds
+          sleep @check_interval
+
+          # Only trigger the resolve if the queue is empty, every other message
+          # on the queue would either cause a resolve or stop the watcher
+          if @message_queue.empty?
+            @message_queue.push(Messages::CHECK_INTERVAL_MESSAGE)
+          end
+
+        end
+        log.info "synapse: zookeeper_dns watcher exited successfully"
+      end
+    end
+
+    def ping?
+      @watcher.alive? && @dns.ping? && @zk.ping?
+    end
+
+    def stop
+      super
+
+      @dns.stop
+      @zk.stop
+    end
+
+    def backends
+      @dns.backends
+    end
+
+    private
+
+    def validate_discovery_opts
+      unless @discovery['method'] == 'zookeeper_dns'
+        raise ArgumentError, "invalid discovery method #{@discovery['method']}"
+      end
+
+      unless @discovery['hosts']
+        raise ArgumentError, "missing or invalid zookeeper host for service #{@name}"
+      end
+
+      unless @discovery['path']
+        raise ArgumentError, "invalid zookeeper path for service #{@name}"
+      end
+    end
+
+    # Method to generate a full config for the children (Dns and Zookeeper)
+    # watchers
+    #
+    # Notes on passing in the default_servers:
+    #
+    #   Setting the default_servers here allows the Zookeeper watcher to return
+    #   a list of backends based on the default servers when it fails to find
+    #   any matching servers.  These are passed on as the discovered backends
+    #   to the DNS watcher, which will then watch them as normal for DNS
+    #   changes.  The default servers can also come into play if none of the
+    #   hostnames from Zookeeper resolve to addresses in the DNS watcher.  This
+    #   should generally result in the expected behavior, but caution should be
+    #   taken when deciding that this is the desired behavior.
+    def mk_child_watcher_opts(discovery_opts)
+      {
+        'name' => @name,
+        'haproxy' => @haproxy,
+        'discovery' => discovery_opts,
+        'default_servers' => @default_servers,
+      }
+    end
+
+    # Override reconfigure! as this class should not explicitly reconfigure
+    # synapse
+    def reconfigure!
+    end
+  end
+end