synapse 0.2.1 → 0.8.0

data/lib/synapse/log.rb

@@ -0,0 +1,24 @@
+module Synapse
+  module Logging
+
+    def log
+      @logger ||= Logging.logger_for(self.class.name)
+    end
+
+    # Use a hash class-ivar to cache a unique Logger per class:
+    @loggers = {}
+
+    class << self
+      def logger_for(classname)
+        @loggers[classname] ||= configure_logger_for(classname)
+      end
+
+      def configure_logger_for(classname)
+        logger = Logger.new(STDERR)
+        logger.level = Logger::INFO unless ENV['DEBUG']
+        logger.progname = classname
+        return logger
+      end
+    end
+  end
+end

data/lib/synapse/service_watcher.rb

@@ -1,7 +1,8 @@
-require_relative "./service_watcher/base"
-require_relative "./service_watcher/zookeeper"
-require_relative "./service_watcher/ec2tag"
-require_relative "./service_watcher/dns"
+require "synapse/service_watcher/base"
+require "synapse/service_watcher/zookeeper"
+require "synapse/service_watcher/ec2tag"
+require "synapse/service_watcher/dns"
+require "synapse/service_watcher/docker"
 
 module Synapse
   class ServiceWatcher
@@ -10,11 +11,14 @@ module Synapse
       'base'=>BaseWatcher,
       'zookeeper'=>ZookeeperWatcher,
       'ec2tag'=>EC2Watcher,
-      'dns' => DnsWatcher
+      'dns' => DnsWatcher,
+      'docker' => DockerWatcher
     }
 
     # the method which actually dispatches watcher creation requests
-    def self.create(opts, synapse)
+    def self.create(name, opts, synapse)
+      opts['name'] = name
+
       raise ArgumentError, "Missing discovery method when trying to create watcher" \
         unless opts.has_key?('discovery') && opts['discovery'].has_key?('method')
 

data/lib/synapse/service_watcher/base.rb

@@ -1,7 +1,6 @@
-
 module Synapse
   class BaseWatcher
-    attr_reader :backends, :name, :listen, :local_port, :server_options
+    attr_reader :name, :backends, :haproxy
 
     def initialize(opts={}, synapse)
       super()
@@ -9,34 +8,57 @@ module Synapse
       @synapse = synapse
 
       # set required service parameters
-      %w{name discovery local_port}.each do |req|
+      %w{name discovery haproxy}.each do |req|
         raise ArgumentError, "missing required option #{req}" unless opts[req]
       end
 
       @name = opts['name']
       @discovery = opts['discovery']
-      @local_port = opts['local_port']
 
-      # optional service parameters
-      @listen = opts['listen'] || []
-      @server_options = opts['server_options'] || ""
-      @default_servers = opts['default_servers'] || []
-      @server_port_override = opts['server_port_override']
+      # the haproxy config
+      @haproxy = opts['haproxy']
+      @haproxy['server_options'] ||= ""
+      @haproxy['server_port_override'] ||= nil
+      %w{backend frontend listen}.each do |sec|
+        @haproxy[sec] ||= []
+      end
 
-      # set initial backends to default servers
+      unless @haproxy.include?('port')
+        log.warn "synapse: service #{name}: haproxy config does not include a port; only backend sections for the service will be created; you must move traffic there manually using configuration in `extra_sections`"
+      end
+
+      # set initial backends to default servers, if any
+      @default_servers = opts['default_servers'] || []
       @backends = @default_servers
 
+      # set a flag used to tell the watchers to exit
+      # this is not used in every watcher
+      @should_exit = false
+
       validate_discovery_opts
     end
 
+    # this should be overridden to actually start your watcher
     def start
       log.info "synapse: starting stub watcher; this means doing nothing at all!"
     end
 
+    # this should be overridden to actually stop your watcher if necessary
+    # if you are running a thread, your loop should run `until @should_exit`
+    def stop
+      log.info "synapse: stopping watcher #{self.name} using default stop handler"
+      @should_exit = true
+    end
+
+    # this should be overridden to do a health check of the watcher
+    def ping?
+      true
+    end
+
     private
     def validate_discovery_opts
       raise ArgumentError, "invalid discovery method '#{@discovery['method']}' for base watcher" \
-        unless @discovery['method'] == 'base' 
+        unless @discovery['method'] == 'base'
 
       log.warn "synapse: warning: a stub watcher with no default servers is pretty useless" if @default_servers.empty?
     end

data/lib/synapse/service_watcher/dns.rb

@@ -1,15 +1,23 @@
-require_relative "./base"
+require "synapse/service_watcher/base"
+require "synapse/log"
 
 require 'thread'
 require 'resolv'
 
 module Synapse
   class DnsWatcher < BaseWatcher
+    include Logging
     def start
       @check_interval = @discovery['check_interval'] || 30.0
       @nameserver = @discovery['nameserver']
 
-      watch
+      @watcher = Thread.new do
+        watch
+      end
+    end
+
+    def ping?
+      !(resolver.getaddresses('airbnb.com').empty?)
     end
 
     private
@@ -21,25 +29,25 @@ module Synapse
     end
 
     def watch
-      @watcher = Thread.new do
-        last_resolution = resolve_servers
-        configure_backends(last_resolution)
-        while true
-          begin
-            start = Time.now
-            current_resolution = resolve_servers
-            unless last_resolution == current_resolution
-              last_resolution = current_resolution
-              configure_backends(last_resolution)
-            end
-
-            sleep_until_next_check(start)
-          rescue => e
-            log.warn "Error in watcher thread: #{e.inspect}"
-            log.warn e.backtrace
+      last_resolution = resolve_servers
+      configure_backends(last_resolution)
+      until @should_exit
+        begin
+          start = Time.now
+          current_resolution = resolve_servers
+          unless last_resolution == current_resolution
+            last_resolution = current_resolution
+            configure_backends(last_resolution)
           end
+
+          sleep_until_next_check(start)
+        rescue => e
+          log.warn "Error in watcher thread: #{e.inspect}"
+          log.warn e.backtrace
         end
       end
+
+      log.info "synapse: dns watcher exited successfully"
     end
 
     def sleep_until_next_check(start_time)
@@ -72,7 +80,7 @@ module Synapse
       new_backends = servers.flat_map do |(server, addresses)|
         addresses.map do |address|
           {
-            'name' => "#{server['name']}-#{[address, server['port']].hash}",
+            'name' => server['name'],
             'host' => address,
             'port' => server['port']
           }
@@ -92,7 +100,7 @@ module Synapse
         log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
         @backends = new_backends
       end
-      @synapse.configure
+      @synapse.reconfigure!
     end
   end
 end

data/lib/synapse/service_watcher/docker.rb

@@ -0,0 +1,108 @@
+require "synapse/service_watcher/base"
+require 'docker'
+
+module Synapse
+  class DockerWatcher < BaseWatcher
+    def start
+      @check_interval = @discovery['check_interval'] || 15.0
+      @watcher = Thread.new do
+        watch
+      end
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'docker'
+      raise ArgumentError, "a non-empty list of servers is required" \
+        if @discovery['servers'].nil? or @discovery['servers'].empty?
+      raise ArgumentError, "non-empty image_name required" \
+        if @discovery['image_name'].nil? or @discovery['image_name'].empty?
+      raise ArgumentError, "container_port required" \
+        if @discovery['container_port'].nil?
+    end
+
+    def watch
+      last_containers = []
+      until @should_exit
+        begin
+          start = Time.now
+          current_containers = containers
+          unless last_containers == current_containers
+            last_containers = current_containers
+            configure_backends(last_containers)
+          end
+
+          sleep_until_next_check(start)
+        rescue => e
+          log.warn "synapse: error in watcher thread: #{e.inspect}"
+          log.warn e.backtrace
+        end
+      end
+
+      log.info "synapse: docker watcher exited successfully"
+    end
+
+    def sleep_until_next_check(start_time)
+      sleep_time = @check_interval - (Time.now - start_time)
+      if sleep_time > 0.0
+        sleep(sleep_time)
+      end
+    end
+
+    def containers
+      backends = @discovery['servers'].map do |server|
+        Docker.url = "http://#{server['host']}:#{server['port'] || 4243}"
+        begin
+          cnts = Docker::Util.parse_json(Docker.connection.get('/containers/json', {}))
+        rescue => e
+          log.warn "synapse: error polling docker host #{Docker.url}: #{e.inspect}"
+          next []
+        end
+        # "Ports" comes through (as of 0.6.5) as a string like "0.0.0.0:49153->6379/tcp, 0.0.0.0:49153->6379/tcp"
+        # Convert string to a map of container port to host port: {"7000"->"49158", "6379": "49159"}
+        cnts.each do |cnt|
+          pairs = cnt["Ports"].split(", ").collect do |v|
+            pair = v.split('->')
+            [ pair[1].rpartition("/").first, pair[0].rpartition(":").last ]
+          end
+          cnt["Ports"] = Hash[pairs]
+        end
+        # Discover containers that match the image/port we're interested in
+        cnts = cnts.find_all do |cnt|
+          cnt["Image"].rpartition(":").first == @discovery["image_name"] \
+            and cnt["Ports"].has_key?(@discovery["container_port"].to_s())
+        end
+        cnts.map do |cnt|
+          {
+            'name' => server['name'],
+            'host' => server['host'],
+            'port' => cnt["Ports"][@discovery["container_port"].to_s()]
+          }
+        end
+      end
+      backends.flatten
+    rescue => e
+      log.warn "synapse: error while polling for containers: #{e.inspect}"
+      []
+    end
+
+    def configure_backends(new_backends)
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name};" \
+            " using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name};" \
+            " using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        @backends = new_backends
+      end
+      @synapse.reconfigure!
+    end
+
+  end
+end

data/lib/synapse/service_watcher/ec2tag.rb

@@ -1,4 +1,4 @@
-require_relative "./base"
+require "synapse/service_watcher/base"
 
 module Synapse
   class EC2Watcher < BaseWatcher

data/lib/synapse/service_watcher/zookeeper.rb

@@ -1,21 +1,34 @@
-require_relative "./base"
+require "synapse/service_watcher/base"
+require "synapse/log"
 
-require_relative "../../gen-rb/endpoint_types"
-require_relative "../../gen-rb/thrift"
 require 'zk'
 
 module Synapse
   class ZookeeperWatcher < BaseWatcher
+    include Logging
     def start
       zk_hosts = @discovery['hosts'].shuffle.join(',')
 
       log.info "synapse: starting ZK watcher #{@name} @ hosts: #{zk_hosts}, path: #{@discovery['path']}"
+      @should_exit = false
       @zk = ZK.new(zk_hosts)
 
-      @deserializer = Thrift::Deserializer.new
+      # call the callback to bootstrap the process
+      watcher_callback.call
+    end
+
+    def stop
+      log.warn "synapse: zookeeper watcher exiting"
+
+      @should_exit = true
+      @watcher.unsubscribe if defined? @watcher
+      @zk.close! if defined? @zk
 
-      watch
-      discover
+      log.info "synapse: zookeeper watcher cleaned up successfully"
+    end
+
+    def ping?
+      @zk.ping?
     end
 
     private
@@ -51,10 +64,9 @@ module Synapse
             log.error "synapse: invalid data in ZK node #{name} at #{@discovery['path']}"
           else
             server_port = @server_port_override ? @server_port_override : port
-            backend_name = "#{name}-#{[host, server_port].hash}"
 
-            log.debug "synapse: discovered backend #{backend_name} at #{host}:#{server_port} for service #{@name}"
-            new_backends << { 'name' => backend_name, 'host' => host, 'port' => server_port}
+            log.debug "synapse: discovered backend #{name} at #{host}:#{server_port} for service #{@name}"
+            new_backends << { 'name' => name, 'host' => host, 'port' => server_port}
           end
         end
       rescue ZK::Exceptions::NoNode
@@ -78,19 +90,21 @@ module Synapse
 
     # sets up zookeeper callbacks if the data at the discovery path changes
     def watch
+      return if @should_exit
+
       @watcher.unsubscribe if defined? @watcher
       @watcher = @zk.register(@discovery['path'], &watcher_callback)
     end
 
     # handles the event that a watched path has changed in zookeeper
     def watcher_callback
-      Proc.new do |event|
+      @callback ||= Proc.new do |event|
         # Set new watcher
         watch
         # Rediscover
         discover
         # send a message to calling class to reconfigure
-        @synapse.configure
+        @synapse.reconfigure!
       end
     end
 
@@ -106,27 +120,10 @@ module Synapse
       return json['host'], json['port']
     end
 
-    # tries to extract a host/port from twitter thrift data
-    def parse_thrift(data)
-      begin
-        service = Twitter::Thrift::ServiceInstance.new
-        @deserializer.deserialize(service, data)
-      rescue Object => o
-        return false
-      end
-      raise "instance thrift data does not have host" if service.serviceEndpoint.host.nil?
-      raise "instance thrift data does not have port" if service.serviceEndpoint.port.nil?
-      return service.serviceEndpoint.host, service.serviceEndpoint.port
-    end
-
     # decode the data at a zookeeper endpoint
     def deserialize_service_instance(data)
       log.debug "synapse: deserializing process data"
 
-      # first, lets try parsing this as thrift
-      host, port = parse_thrift(data)
-      return host, port if host
-
       # if that does not work, try json
       host, port = parse_json(data)
       return host, port if host

data/lib/synapse/version.rb

@@ -1,3 +1,3 @@
 module Synapse
-  VERSION = "0.2.1"
+  VERSION = "0.8.0"
 end

data/spec/spec_helper.rb

@@ -4,9 +4,9 @@
 # loaded once.
 #
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
-require_relative '../lib/synapse'
+require '../lib/synapse'
 require 'pry'
-require_relative 'support/config'
+require 'support/config'
 
 RSpec.configure do |config|
   config.treat_symbols_as_metadata_keys_with_true_values = true