synapse-aurora 0.11.2

data/lib/synapse/log.rb

@@ -0,0 +1,24 @@
+module Synapse
+  module Logging
+
+    def log
+      @logger ||= Logging.logger_for(self.class.name)
+    end
+
+    # Use a hash class-ivar to cache a unique Logger per class:
+    @loggers = {}
+
+    class << self
+      def logger_for(classname)
+        @loggers[classname] ||= configure_logger_for(classname)
+      end
+
+      def configure_logger_for(classname)
+        logger = Logger.new(STDERR)
+        logger.level = Logger::INFO unless ENV['DEBUG']
+        logger.progname = classname
+        return logger
+      end
+    end
+  end
+end

data/lib/synapse/service_watcher.rb

@@ -0,0 +1,36 @@
+require "synapse/service_watcher/base"
+require "synapse/service_watcher/zookeeper"
+require "synapse/service_watcher/ec2tag"
+require "synapse/service_watcher/dns"
+require "synapse/service_watcher/docker"
+require "synapse/service_watcher/zookeeper_dns"
+require "synapse/service_watcher/zookeeper_aurora"
+
+module Synapse
+  class ServiceWatcher
+
+    @watchers = {
+      'base' => BaseWatcher,
+      'zookeeper' => ZookeeperWatcher,
+      'ec2tag' => EC2Watcher,
+      'dns' => DnsWatcher,
+      'docker' => DockerWatcher,
+      'zookeeper_dns' => ZookeeperDnsWatcher,
+      'zookeeper_aurora' => ZookeeperAuroraWatcher
+    }
+
+    # the method which actually dispatches watcher creation requests
+    def self.create(name, opts, synapse)
+      opts['name'] = name
+
+      raise ArgumentError, "Missing discovery method when trying to create watcher" \
+        unless opts.has_key?('discovery') && opts['discovery'].has_key?('method')
+
+      discovery_method = opts['discovery']['method']
+      raise ArgumentError, "Invalid discovery method #{discovery_method}" \
+        unless @watchers.has_key?(discovery_method)
+
+      return @watchers[discovery_method].new(opts, synapse)
+    end
+  end
+end

data/lib/synapse/service_watcher/base.rb

@@ -0,0 +1,109 @@
+require 'synapse/log'
+
+module Synapse
+  class BaseWatcher
+    include Logging
+
+    LEADER_WARN_INTERVAL = 30
+
+    attr_reader :name, :haproxy
+
+    def initialize(opts={}, synapse)
+      super()
+
+      @synapse = synapse
+
+      # set required service parameters
+      %w{name discovery haproxy}.each do |req|
+        raise ArgumentError, "missing required option #{req}" unless opts[req]
+      end
+
+      @name = opts['name']
+      @discovery = opts['discovery']
+
+      @leader_election = opts['leader_election'] || false
+      @leader_last_warn = Time.now - LEADER_WARN_INTERVAL
+
+      # the haproxy config
+      @haproxy = opts['haproxy']
+      @haproxy['server_options'] ||= ""
+      @haproxy['server_port_override'] ||= nil
+      %w{backend frontend listen}.each do |sec|
+        @haproxy[sec] ||= []
+      end
+
+      unless @haproxy.include?('port')
+        log.warn "synapse: service #{name}: haproxy config does not include a port; only backend sections for the service will be created; you must move traffic there manually using configuration in `extra_sections`"
+      end
+
+      # set initial backends to default servers, if any
+      @default_servers = opts['default_servers'] || []
+      @backends = @default_servers
+
+      @keep_default_servers = opts['keep_default_servers'] || false
+
+      # set a flag used to tell the watchers to exit
+      # this is not used in every watcher
+      @should_exit = false
+
+      validate_discovery_opts
+    end
+
+    # this should be overridden to actually start your watcher
+    def start
+      log.info "synapse: starting stub watcher; this means doing nothing at all!"
+    end
+
+    # this should be overridden to actually stop your watcher if necessary
+    # if you are running a thread, your loop should run `until @should_exit`
+    def stop
+      log.info "synapse: stopping watcher #{self.name} using default stop handler"
+      @should_exit = true
+    end
+
+    # this should be overridden to do a health check of the watcher
+    def ping?
+      true
+    end
+
+    def backends
+      if @leader_election
+        if @backends.all?{|b| b.key?('id') && b['id']}
+          smallest = @backends.sort_by{ |b| b['id']}.first
+          log.debug "synapse: leader election chose one of #{@backends.count} backends " \
+            "(#{smallest['host']}:#{smallest['port']} with id #{smallest['id']})"
+
+          return [smallest]
+        elsif (Time.now - @leader_last_warn) > LEADER_WARN_INTERVAL
+          log.warn "synapse: service #{@name}: leader election failed; not all backends include an id"
+          @leader_last_warn = Time.now
+        end
+
+        # if leader election fails, return no backends
+        return []
+      end
+
+      return @backends
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method '#{@discovery['method']}' for base watcher" \
+        unless @discovery['method'] == 'base'
+
+      log.warn "synapse: warning: a stub watcher with no default servers is pretty useless" if @default_servers.empty?
+    end
+
+    def set_backends(new_backends)
+      if @keep_default_servers
+        @backends = @default_servers + new_backends
+      else
+        @backends = new_backends
+      end
+    end
+
+    def reconfigure!
+      @synapse.reconfigure!
+    end
+  end
+end

data/lib/synapse/service_watcher/dns.rb

@@ -0,0 +1,109 @@
+require "synapse/service_watcher/base"
+
+require 'thread'
+require 'resolv'
+
+module Synapse
+  class DnsWatcher < BaseWatcher
+    def start
+      @check_interval = @discovery['check_interval'] || 30.0
+      @nameserver = @discovery['nameserver']
+
+      @watcher = Thread.new do
+        watch
+      end
+    end
+
+    def ping?
+      @watcher.alive? && !(resolver.getaddresses('airbnb.com').empty?)
+    end
+
+    def discovery_servers
+      @discovery['servers']
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'dns'
+      raise ArgumentError, "a non-empty list of servers is required" \
+        if discovery_servers.empty?
+    end
+
+    def watch
+      last_resolution = resolve_servers
+      configure_backends(last_resolution)
+      until @should_exit
+        begin
+          start = Time.now
+          current_resolution = resolve_servers
+          unless last_resolution == current_resolution
+            last_resolution = current_resolution
+            configure_backends(last_resolution)
+          end
+
+          sleep_until_next_check(start)
+        rescue => e
+          log.warn "Error in watcher thread: #{e.inspect}"
+          log.warn e.backtrace
+        end
+      end
+
+      log.info "synapse: dns watcher exited successfully"
+    end
+
+    def sleep_until_next_check(start_time)
+      sleep_time = @check_interval - (Time.now - start_time)
+      if sleep_time > 0.0
+        sleep(sleep_time)
+      end
+    end
+
+    def resolve_servers
+      resolver.tap do |dns|
+        resolution = discovery_servers.map do |server|
+          addresses = dns.getaddresses(server['host']).map(&:to_s)
+          [server, addresses.sort]
+        end
+
+        return resolution
+      end
+    rescue => e
+      log.warn "Error while resolving host names: #{e.inspect}"
+      []
+    end
+
+    def resolver
+      args = [{:nameserver => @nameserver}] if @nameserver
+      Resolv::DNS.open(*args)
+    end
+
+    def configure_backends(servers)
+      new_backends = servers.flat_map do |(server, addresses)|
+        addresses.map do |address|
+          {
+            'host' => address,
+            'port' => server['port'],
+            'name' => server['name'],
+          }
+        end
+      end
+
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name};" \
+            " using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name};" \
+            " using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        set_backends(new_backends)
+      end
+
+      reconfigure!
+    end
+  end
+end

data/lib/synapse/service_watcher/docker.rb

@@ -0,0 +1,120 @@
+require "synapse/service_watcher/base"
+require 'docker'
+
+module Synapse
+  class DockerWatcher < BaseWatcher
+    def start
+      @check_interval = @discovery['check_interval'] || 15.0
+      @watcher = Thread.new do
+        watch
+      end
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'docker'
+      raise ArgumentError, "a non-empty list of servers is required" \
+        if @discovery['servers'].nil? or @discovery['servers'].empty?
+      raise ArgumentError, "non-empty image_name required" \
+        if @discovery['image_name'].nil? or @discovery['image_name'].empty?
+      raise ArgumentError, "container_port required" \
+        if @discovery['container_port'].nil?
+    end
+
+    def watch
+      last_containers = []
+      until @should_exit
+        begin
+          start = Time.now
+          current_containers = containers
+          unless last_containers == current_containers
+            last_containers = current_containers
+            configure_backends(last_containers)
+          end
+
+          sleep_until_next_check(start)
+        rescue Exception => e
+          log.warn "synapse: error in watcher thread: #{e.inspect}"
+          log.warn e.backtrace
+        end
+      end
+
+      log.info "synapse: docker watcher exited successfully"
+    end
+
+    def sleep_until_next_check(start_time)
+      sleep_time = @check_interval - (Time.now - start_time)
+      if sleep_time > 0.0
+        sleep(sleep_time)
+      end
+    end
+
+    def rewrite_container_ports(ports)
+      pairs = []
+      if ports.is_a?(String)
+        # "Ports" comes through (as of 0.6.5) as a string like "0.0.0.0:49153->6379/tcp, 0.0.0.0:49153->6379/tcp"
+        # Convert string to a map of container port to host port: {"7000"->"49158", "6379": "49159"}
+        pairs = ports.split(", ").collect do |v|
+          pair = v.split('->')
+          [ pair[1].rpartition("/").first, pair[0].rpartition(":").last ]
+        end
+      elsif ports.is_a?(Array)
+        # New style API, ports is an array of hashes, with numeric values (or nil if no ports forwarded)
+        pairs = ports.collect do |v|
+          [v['PrivatePort'].to_s, v['PublicPort'].to_s]
+        end
+      end
+      Hash[pairs]
+    end
+
+    def containers
+      backends = @discovery['servers'].map do |server|
+        Docker.url = "http://#{server['host']}:#{server['port'] || 4243}"
+        begin
+          cnts = Docker::Util.parse_json(Docker.connection.get('/containers/json', {}))
+        rescue => e
+          log.warn "synapse: error polling docker host #{Docker.url}: #{e.inspect}"
+          next []
+        end
+        cnts.each do |cnt|
+          cnt['Ports'] = rewrite_container_ports cnt['Ports']
+        end
+        # Discover containers that match the image/port we're interested in
+        cnts = cnts.find_all do |cnt|
+          cnt["Image"].rpartition(":").first == @discovery["image_name"] \
+            and cnt["Ports"].has_key?(@discovery["container_port"].to_s())
+        end
+        cnts.map do |cnt|
+          {
+            'name' => server['name'],
+            'host' => server['host'],
+            'port' => cnt["Ports"][@discovery["container_port"].to_s()]
+          }
+        end
+      end
+      backends.flatten
+    rescue => e
+      log.warn "synapse: error while polling for containers: #{e.inspect}"
+      []
+    end
+
+    def configure_backends(new_backends)
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name};" \
+            " using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name};" \
+            " using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        set_backends(new_backends)
+      end
+      reconfigure!
+    end
+
+  end
+end

data/lib/synapse/service_watcher/ec2tag.rb

@@ -0,0 +1,133 @@
+require 'synapse/service_watcher/base'
+require 'aws-sdk'
+
+module Synapse
+  class EC2Watcher < BaseWatcher
+
+    attr_reader :check_interval
+
+    def start
+      region = @discovery['aws_region'] || ENV['AWS_REGION']
+      log.info "Connecting to EC2 region: #{region}"
+
+      @ec2 = AWS::EC2.new(
+        region:            region,
+        access_key_id:     @discovery['aws_access_key_id']     || ENV['AWS_ACCESS_KEY_ID'],
+        secret_access_key: @discovery['aws_secret_access_key'] || ENV['AWS_SECRET_ACCESS_KEY'] )
+
+      @check_interval = @discovery['check_interval'] || 15.0
+
+      log.info "synapse: ec2tag watcher looking for instances " +
+        "tagged with #{@discovery['tag_name']}=#{@discovery['tag_value']}"
+
+      @watcher = Thread.new { watch }
+    end
+
+    private
+
+    def validate_discovery_opts
+      # Required, via options only.
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'ec2tag'
+      raise ArgumentError, "aws tag name is required for service #{@name}" \
+        unless @discovery['tag_name']
+      raise ArgumentError, "aws tag value required for service #{@name}" \
+        unless @discovery['tag_value']
+
+      # As we're only looking up instances with hostnames/IPs, need to
+      # be explicitly told which port the service we're balancing for listens on.
+      unless @haproxy['server_port_override']
+        raise ArgumentError,
+          "Missing server_port_override for service #{@name} - which port are backends listening on?"
+      end
+
+      unless @haproxy['server_port_override'].match(/^\d+$/)
+        raise ArgumentError, "Invalid server_port_override value"
+      end
+
+      # Required, but can use well-known environment variables.
+      %w[aws_access_key_id aws_secret_access_key aws_region].each do |attr|
+        unless (@discovery[attr] || ENV[attr.upcase])
+          raise ArgumentError, "Missing #{attr} option or #{attr.upcase} environment variable"
+        end
+      end
+    end
+
+    def watch
+      last_backends = []
+      until @should_exit
+        begin
+          start = Time.now
+          current_backends = discover_instances
+
+          if last_backends != current_backends
+            log.info "synapse: ec2tag watcher backends have changed."
+            last_backends = current_backends
+            configure_backends(current_backends)
+          else
+            log.info "synapse: ec2tag watcher backends are unchanged."
+          end
+
+          sleep_until_next_check(start)
+        rescue Exception => e
+          log.warn "synapse: error in ec2tag watcher thread: #{e.inspect}"
+          log.warn e.backtrace
+        end
+      end
+
+      log.info "synapse: ec2tag watcher exited successfully"
+    end
+
+    def sleep_until_next_check(start_time)
+      sleep_time = check_interval - (Time.now - start_time)
+      if sleep_time > 0.0
+        sleep(sleep_time)
+      end
+    end
+
+    def discover_instances
+      AWS.memoize do
+        instances = instances_with_tags(@discovery['tag_name'], @discovery['tag_value'])
+
+        new_backends = []
+
+        # choice of private_dns_name, dns_name, private_ip_address or
+        # ip_address, for now, just stick with the private fields.
+        instances.each do |instance|
+          new_backends << {
+            'name' => instance.private_dns_name,
+            'host' => instance.private_ip_address,
+            'port' => @haproxy['server_port_override'],
+          }
+        end
+
+        new_backends
+      end
+    end
+
+    def instances_with_tags(tag_name, tag_value)
+      @ec2.instances
+        .tagged(tag_name)
+        .tagged_values(tag_value)
+        .select { |i| i.status == :running }
+    end
+
+    def configure_backends(new_backends)
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name};" \
+            " using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name};" \
+            " using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        @backends = new_backends
+      end
+      @synapse.reconfigure!
+    end
+  end
+end
+