democracyworks-synapse 0.9.2

data/lib/synapse/log.rb

@@ -0,0 +1,24 @@
+module Synapse
+  module Logging
+
+    def log
+      @logger ||= Logging.logger_for(self.class.name)
+    end
+
+    # Use a hash class-ivar to cache a unique Logger per class:
+    @loggers = {}
+
+    class << self
+      def logger_for(classname)
+        @loggers[classname] ||= configure_logger_for(classname)
+      end
+
+      def configure_logger_for(classname)
+        logger = Logger.new(STDERR)
+        logger.level = Logger::INFO unless ENV['DEBUG']
+        logger.progname = classname
+        return logger
+      end
+    end
+  end
+end

data/lib/synapse/service_watcher.rb

@@ -0,0 +1,32 @@
+require "synapse/service_watcher/base"
+require "synapse/service_watcher/zookeeper"
+require "synapse/service_watcher/ec2tag"
+require "synapse/service_watcher/dns"
+require "synapse/service_watcher/docker"
+
+module Synapse
+  class ServiceWatcher
+
+    @watchers = {
+      'base'=>BaseWatcher,
+      'zookeeper'=>ZookeeperWatcher,
+      'ec2tag'=>EC2Watcher,
+      'dns' => DnsWatcher,
+      'docker' => DockerWatcher
+    }
+
+    # the method which actually dispatches watcher creation requests
+    def self.create(name, opts, synapse)
+      opts['name'] = name
+
+      raise ArgumentError, "Missing discovery method when trying to create watcher" \
+        unless opts.has_key?('discovery') && opts['discovery'].has_key?('method')
+
+      discovery_method = opts['discovery']['method']
+      raise ArgumentError, "Invalid discovery method #{discovery_method}" \
+        unless @watchers.has_key?(discovery_method)
+
+      return @watchers[discovery_method].new(opts, synapse)
+    end
+  end
+end

data/lib/synapse/service_watcher/base.rb

@@ -0,0 +1,105 @@
+require 'synapse/log'
+
+module Synapse
+  class BaseWatcher
+    include Logging
+
+    LEADER_WARN_INTERVAL = 30
+
+    attr_reader :name, :haproxy
+
+    def initialize(opts={}, synapse)
+      super()
+
+      @synapse = synapse
+
+      # set required service parameters
+      %w{name discovery haproxy}.each do |req|
+        raise ArgumentError, "missing required option #{req}" unless opts[req]
+      end
+
+      @name = opts['name']
+      @discovery = opts['discovery']
+
+      @leader_election = opts['leader_election'] || false
+      @leader_last_warn = Time.now - LEADER_WARN_INTERVAL
+
+      # the haproxy config
+      @haproxy = opts['haproxy']
+      @haproxy['server_options'] ||= ""
+      @haproxy['server_port_override'] ||= nil
+      %w{backend frontend listen}.each do |sec|
+        @haproxy[sec] ||= []
+      end
+
+      unless @haproxy.include?('port')
+        log.warn "synapse: service #{name}: haproxy config does not include a port; only backend sections for the service will be created; you must move traffic there manually using configuration in `extra_sections`"
+      end
+
+      # set initial backends to default servers, if any
+      @default_servers = opts['default_servers'] || []
+      @backends = @default_servers
+
+      @keep_default_servers = opts['keep_default_servers'] || false
+
+      # set a flag used to tell the watchers to exit
+      # this is not used in every watcher
+      @should_exit = false
+
+      validate_discovery_opts
+    end
+
+    # this should be overridden to actually start your watcher
+    def start
+      log.info "synapse: starting stub watcher; this means doing nothing at all!"
+    end
+
+    # this should be overridden to actually stop your watcher if necessary
+    # if you are running a thread, your loop should run `until @should_exit`
+    def stop
+      log.info "synapse: stopping watcher #{self.name} using default stop handler"
+      @should_exit = true
+    end
+
+    # this should be overridden to do a health check of the watcher
+    def ping?
+      true
+    end
+
+    def backends
+      if @leader_election
+        if @backends.all?{|b| b.key?('id') && b['id']}
+          smallest = @backends.sort_by{ |b| b['id']}.first
+          log.debug "synapse: leader election chose one of #{@backends.count} backends " \
+            "(#{smallest['host']}:#{smallest['port']} with id #{smallest['id']})"
+
+          return [smallest]
+        elsif (Time.now - @leader_last_warn) > LEADER_WARN_INTERVAL
+          log.warn "synapse: service #{@name}: leader election failed; not all backends include an id"
+          @leader_last_warn = Time.now
+        end
+
+        # if leader election fails, return no backends
+        return []
+      end
+
+      return @backends
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method '#{@discovery['method']}' for base watcher" \
+        unless @discovery['method'] == 'base'
+
+      log.warn "synapse: warning: a stub watcher with no default servers is pretty useless" if @default_servers.empty?
+    end
+
+    def set_backends(new_backends)
+      if @keep_default_servers
+        @backends = @default_servers + new_backends
+      else
+        @backends = new_backends
+      end
+    end
+  end
+end

data/lib/synapse/service_watcher/dns.rb

@@ -0,0 +1,103 @@
+require "synapse/service_watcher/base"
+
+require 'thread'
+require 'resolv'
+
+module Synapse
+  class DnsWatcher < BaseWatcher
+    def start
+      @check_interval = @discovery['check_interval'] || 30.0
+      @nameserver = @discovery['nameserver']
+
+      @watcher = Thread.new do
+        watch
+      end
+    end
+
+    def ping?
+      !(resolver.getaddresses('airbnb.com').empty?)
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'dns'
+      raise ArgumentError, "a non-empty list of servers is required" \
+        if @discovery['servers'].empty?
+    end
+
+    def watch
+      last_resolution = resolve_servers
+      configure_backends(last_resolution)
+      until @should_exit
+        begin
+          start = Time.now
+          current_resolution = resolve_servers
+          unless last_resolution == current_resolution
+            last_resolution = current_resolution
+            configure_backends(last_resolution)
+          end
+
+          sleep_until_next_check(start)
+        rescue => e
+          log.warn "Error in watcher thread: #{e.inspect}"
+          log.warn e.backtrace
+        end
+      end
+
+      log.info "synapse: dns watcher exited successfully"
+    end
+
+    def sleep_until_next_check(start_time)
+      sleep_time = @check_interval - (Time.now - start_time)
+      if sleep_time > 0.0
+        sleep(sleep_time)
+      end
+    end
+
+    def resolve_servers
+      resolver.tap do |dns|
+        resolution = @discovery['servers'].map do |server|
+          addresses = dns.getaddresses(server['host']).map(&:to_s)
+          [server, addresses.sort]
+        end
+
+        return resolution
+      end
+    rescue => e
+      log.warn "Error while resolving host names: #{e.inspect}"
+      []
+    end
+
+    def resolver
+      args = [{:nameserver => @nameserver}] if @nameserver
+      Resolv::DNS.open(*args)
+    end
+
+    def configure_backends(servers)
+      new_backends = servers.flat_map do |(server, addresses)|
+        addresses.map do |address|
+          {
+            'host' => address,
+            'port' => server['port']
+          }
+        end
+      end
+
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name};" \
+            " using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name};" \
+            " using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        set_backends(new_backends)
+      end
+      @synapse.reconfigure!
+    end
+  end
+end

data/lib/synapse/service_watcher/docker.rb

@@ -0,0 +1,115 @@
+require "synapse/service_watcher/base"
+require 'docker'
+
+module Synapse
+  class DockerWatcher < BaseWatcher
+    def start
+      @check_interval = @discovery['check_interval'] || 15.0
+      @watcher = Thread.new do
+        watch
+      end
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'docker'
+      raise ArgumentError, "a non-empty list of servers is required" \
+        if @discovery['servers'].nil? or @discovery['servers'].empty?
+      raise ArgumentError, "non-empty image_name required" \
+        if @discovery['image_name'].nil? or @discovery['image_name'].empty?
+      raise ArgumentError, "container_port required" \
+        if @discovery['container_port'].nil?
+    end
+
+    def watch
+      last_containers = []
+      until @should_exit
+        begin
+          start = Time.now
+          current_containers = containers
+          unless last_containers == current_containers
+            last_containers = current_containers
+            configure_backends(last_containers)
+          end
+
+          sleep_until_next_check(start)
+        rescue Exception => e
+          log.warn "synapse: error in watcher thread: #{e.inspect}"
+          log.warn e.backtrace
+        end
+      end
+
+      log.info "synapse: docker watcher exited successfully"
+    end
+
+    def sleep_until_next_check(start_time)
+      sleep_time = @check_interval - (Time.now - start_time)
+      if sleep_time > 0.0
+        sleep(sleep_time)
+      end
+    end
+
+    def containers
+      backends = @discovery['servers'].map do |server|
+        Docker.url = "http://#{server['host']}:#{server['port'] || 4243}"
+        begin
+          cnts = Docker::Util.parse_json(Docker.connection.get('/containers/json', {}))
+        rescue => e
+          log.warn "synapse: error polling docker host #{Docker.url}: #{e.inspect}"
+          next []
+        end
+        cnts.each do |cnt|
+          pairs = nil
+          if cnt["Ports"].is_a?(String)
+            # "Ports" comes through (as of 0.6.5) as a string like "0.0.0.0:49153->6379/tcp, 0.0.0.0:49153->6379/tcp"
+            # Convert string to a map of container port to host port: {"7000"->"49158", "6379": "49159"}
+            pairs = cnt["Ports"].split(", ").collect do |v|
+              pair = v.split('->')
+              [ pair[1].rpartition("/").first, pair[0].rpartition(":").last ]
+            end
+          else
+            pairs = cnt["Ports"].collect do |v|
+              [v['PrivatePort'].to_s, v['PublicPort'].to_s]
+            end
+          end
+          cnt["Ports"] = Hash[pairs]
+        end
+        # Discover containers that match the image/port we're interested in
+        cnts = cnts.find_all do |cnt|
+          cnt["Image"].rpartition(":").first == @discovery["image_name"] \
+            and cnt["Ports"].has_key?(@discovery["container_port"].to_s())
+        end
+        cnts.map do |cnt|
+          {
+            'name' => server['name'],
+            'host' => server['host'],
+            'port' => cnt["Ports"][@discovery["container_port"].to_s()]
+          }
+        end
+      end
+      backends.flatten
+    rescue => e
+      log.warn "synapse: error while polling for containers: #{e.inspect}"
+      []
+    end
+
+    def configure_backends(new_backends)
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name};" \
+            " using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name};" \
+            " using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        set_backends(new_backends)
+      end
+      @synapse.reconfigure!
+    end
+
+  end
+end

data/lib/synapse/service_watcher/ec2tag.rb

@@ -0,0 +1,26 @@
+require "synapse/service_watcher/base"
+
+module Synapse
+  class EC2Watcher < BaseWatcher
+    def start
+      # connect to ec2
+      # find all servers whose @discovery['tag_name'] matches @discovery['tag_value']
+      # call @synapse.configure
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'ec2tag' 
+      raise ArgumentError, "a `server_port_override` option is required for ec2tag watchers" \
+        unless @server_port_override
+      raise ArgumentError, "missing aws credentials for service #{@name}" \
+        unless (@discovery['aws_key'] && @discovery['aws_secret'])
+      raise ArgumentError, "aws tag name is required for service #{@name}" \
+        unless @discovery['tag_name']
+      raise ArgumentError, "aws tag value required for service #{@name}" \
+        unless @discovery['tag_value']
+    end
+  end
+end
+

data/lib/synapse/service_watcher/zookeeper.rb

@@ -0,0 +1,127 @@
+require "synapse/service_watcher/base"
+
+require 'zk'
+
+module Synapse
+  class ZookeeperWatcher < BaseWatcher
+    NUMBERS_RE = /^\d+$/
+
+    def start
+      zk_hosts = @discovery['hosts'].shuffle.join(',')
+
+      log.info "synapse: starting ZK watcher #{@name} @ hosts: #{zk_hosts}, path: #{@discovery['path']}"
+      @should_exit = false
+      @zk = ZK.new(zk_hosts)
+
+      # call the callback to bootstrap the process
+      watcher_callback.call
+    end
+
+    def stop
+      log.warn "synapse: zookeeper watcher exiting"
+
+      @should_exit = true
+      @watcher.unsubscribe if defined? @watcher
+      @zk.close! if defined? @zk
+
+      log.info "synapse: zookeeper watcher cleaned up successfully"
+    end
+
+    def ping?
+      @zk.ping?
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'zookeeper'
+      raise ArgumentError, "missing or invalid zookeeper host for service #{@name}" \
+        unless @discovery['hosts']
+      raise ArgumentError, "invalid zookeeper path for service #{@name}" \
+        unless @discovery['path']
+    end
+
+    # helper method that ensures that the discovery path exists
+    def create(path)
+      log.debug "synapse: creating ZK path: #{path}"
+      # recurse if the parent node does not exist
+      create File.dirname(path) unless @zk.exists? File.dirname(path)
+      @zk.create(path, ignore: :node_exists)
+    end
+
+    # find the current backends at the discovery path; sets @backends
+    def discover
+      log.info "synapse: discovering backends for service #{@name}"
+
+      new_backends = []
+      begin
+        @zk.children(@discovery['path'], :watch => true).each do |id|
+          node = @zk.get("#{@discovery['path']}/#{id}")
+
+          begin
+            host, port, name = deserialize_service_instance(node.first)
+          rescue StandardError => e
+            log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
+          else
+            server_port = @server_port_override ? @server_port_override : port
+
+            # find the numberic id in the node name; used for leader elections if enabled
+            numeric_id = id.split('_').last
+            numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
+
+            log.debug "synapse: discovered backend #{name} at #{host}:#{server_port} for service #{@name}"
+            new_backends << { 'name' => name, 'host' => host, 'port' => server_port, 'id' => numeric_id}
+          end
+        end
+      rescue ZK::Exceptions::NoNode
+        # the path must exist, otherwise watch callbacks will not work
+        create(@discovery['path'])
+        retry
+      end
+
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name}; using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name}; using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        set_backends(new_backends)
+      end
+    end
+
+    # sets up zookeeper callbacks if the data at the discovery path changes
+    def watch
+      return if @should_exit
+
+      @watcher.unsubscribe if defined? @watcher
+      @watcher = @zk.register(@discovery['path'], &watcher_callback)
+    end
+
+    # handles the event that a watched path has changed in zookeeper
+    def watcher_callback
+      @callback ||= Proc.new do |event|
+        # Set new watcher
+        watch
+        # Rediscover
+        discover
+        # send a message to calling class to reconfigure
+        @synapse.reconfigure!
+      end
+    end
+
+    # decode the data at a zookeeper endpoint
+    def deserialize_service_instance(data)
+      log.debug "synapse: deserializing process data"
+      decoded = JSON.parse(data)
+
+      host = decoded['host'] || (raise ValueError, 'instance json data does not have host key')
+      port = decoded['port'] || (raise ValueError, 'instance json data does not have port key')
+      name = decoded['name'] || nil
+
+      return host, port, name
+    end
+  end
+end