synapse 0.11.1 → 0.12.1

data/lib/synapse/haproxy.rb

@@ -1,10 +1,12 @@
+require 'fileutils'
+require 'json'
 require 'synapse/log'
 require 'socket'
 
 module Synapse
   class Haproxy
     include Logging
-    attr_reader :opts
+    attr_reader :opts, :name
 
     # these come from the documentation for haproxy 1.5
     # http://haproxy.1wt.eu/download/1.5/doc/configuration.txt
@@ -43,6 +45,7 @@ module Synapse
         "option abortonclose",
         "option accept-invalid-http-response",
         "option allbackups",
+        "option allredisp",
         "option checkcache",
         "option forceclose",
         "option forwardfor",
@@ -173,6 +176,7 @@ module Synapse
         "option accept-invalid-http-request",
         "option accept-invalid-http-response",
         "option allbackups",
+        "option allredisp",
         "option checkcache",
         "option clitcpka",
         "option contstats",
@@ -386,6 +390,7 @@ module Synapse
         "option accept-invalid-http-request",
         "option accept-invalid-http-response",
         "option allbackups",
+        "option allredisp",
         "option checkcache",
         "option clitcpka",
         "option contstats",
@@ -523,20 +528,53 @@ module Synapse
       end
 
       @opts = opts
+      @name = 'haproxy'
+
+      @opts['do_writes'] = true unless @opts.key?('do_writes')
+      @opts['do_socket'] = true unless @opts.key?('do_socket')
+      @opts['do_reloads'] = true unless @opts.key?('do_reloads')
 
       # how to restart haproxy
-      @restart_interval = 2
+      @restart_interval = @opts.fetch('restart_interval', 2).to_i
+      @restart_jitter = @opts.fetch('restart_jitter', 0).to_f
       @restart_required = true
-      @last_restart = Time.new(0)
+
+      # virtual clock bookkeeping for controlling how often haproxy restarts
+      @time = 0
+      @next_restart = @time
 
       # a place to store the parsed haproxy config from each watcher
       @watcher_configs = {}
+
+      @state_file_path = @opts['state_file_path']
+      @state_file_ttl = @opts.fetch('state_file_ttl', 60 * 60 * 24).to_i
+      @seen = {}
+
+      unless @state_file_path.nil?
+        begin
+          @seen = JSON.load(File.read(@state_file_path))
+        rescue StandardError => e
+          # It's ok if the state file doesn't exist
+        end
+      end
+    end
+
+    def tick(watchers)
+      if @time % 60 == 0 && !@state_file_path.nil?
+        update_state_file(watchers)
+      end
+
+      @time += 1
+
+      # We potentially have to restart if the restart was rate limited
+      # in the original call to update_config
+      restart if @opts['do_reloads'] && @restart_required
     end
 
     def update_config(watchers)
       # if we support updating backends, try that whenever possible
       if @opts['do_socket']
-        update_backends(watchers) unless @restart_required
+        update_backends(watchers)
       else
         @restart_required = true
       end
@@ -655,17 +693,35 @@ module Synapse
     end
 
     def generate_backend_stanza(watcher, config)
+      backends = {}
+
+      # The ordering here is important.  First we add all the backends in the
+      # disabled state...
+      @seen.fetch(watcher.name, []).each do |backend_name, backend|
+        backends[backend_name] = backend.merge('enabled' => false)
+      end
+
+      # ... and then we overwite any backends that the watchers know about,
+      # setting the enabled state.
+      watcher.backends.each do |backend|
+        backend_name = construct_name(backend)
+        backends[backend_name] = backend.merge('enabled' => true)
+      end
+
       if watcher.backends.empty?
-        log.warn "synapse: no backends found for watcher #{watcher.name}"
+        log.debug "synapse: no backends found for watcher #{watcher.name}"
       end
 
       stanza = [
         "\nbackend #{watcher.name}",
         config.map {|c| "\t#{c}"},
-        watcher.backends.shuffle.map {|backend|
-          backend_name = construct_name(backend)
-          "\tserver #{backend_name} #{backend['host']}:#{backend['port']} " \
-            "cookie #{backend_name} #{watcher.haproxy['server_options']}" }
+        backends.keys.shuffle.map {|backend_name|
+          backend = backends[backend_name]
+          b = "\tserver #{backend_name} #{backend['host']}:#{backend['port']}"
+          b = "#{b} cookie #{backend_name}" unless config.include?('mode tcp')
+          b = "#{b} #{watcher.haproxy['server_options']}"
+          b = "#{b} disabled" unless backend['enabled']
+          b }
       ]
     end
 
@@ -702,27 +758,26 @@ module Synapse
         next if watcher.backends.empty?
 
         unless cur_backends.include? watcher.name
-          log.debug "synapse: restart required because we added new section #{watcher.name}"
+          log.info "synapse: restart required because we added new section #{watcher.name}"
           @restart_required = true
-          return
+          next
         end
 
         watcher.backends.each do |backend|
           backend_name = construct_name(backend)
-          unless cur_backends[watcher.name].include? backend_name
-            log.debug "synapse: restart required because we have a new backend #{watcher.name}/#{backend_name}"
+          if cur_backends[watcher.name].include? backend_name
+            enabled_backends[watcher.name] << backend_name
+          else
+            log.info "synapse: restart required because we have a new backend #{watcher.name}/#{backend_name}"
             @restart_required = true
-            return
           end
-
-          enabled_backends[watcher.name] << backend_name
         end
       end
 
       # actually enable the enabled backends, and disable the disabled ones
       cur_backends.each do |section, backends|
         backends.each do |backend|
-          if enabled_backends[section].include? backend
+          if enabled_backends.fetch(section, []).include? backend
             command = "enable server #{section}/#{backend}\n"
           else
             command = "disable server #{section}/#{backend}\n"
@@ -736,12 +791,10 @@ module Synapse
           rescue StandardError => e
             log.warn "synapse: unknown error writing to socket"
             @restart_required = true
-            return
           else
             unless output == "\n"
               log.warn "synapse: socket command #{command} failed: #{output}"
               @restart_required = true
-              return
             end
           end
         end
@@ -767,18 +820,24 @@ module Synapse
       end
     end
 
-    # restarts haproxy
+    # restarts haproxy if the time is right
     def restart
-      # sleep if we restarted too recently
-      delay = (@last_restart - Time.now) + @restart_interval
-      sleep(delay) if delay > 0
+      if @time < @next_restart
+        log.info "synapse: at time #{@time} waiting until #{@next_restart} to restart"
+        return
+      end
+
+      @next_restart = @time + @restart_interval
+      @next_restart += rand(@restart_jitter * @restart_interval + 1)
 
       # do the actual restart
       res = `#{opts['reload_command']}`.chomp
-      raise "failed to reload haproxy via #{opts['reload_command']}: #{res}" unless $?.success?
+      unless $?.success?
+        log.error "failed to reload haproxy via #{opts['reload_command']}: #{res}"
+        return
+      end
       log.info "synapse: restarted haproxy"
 
-      @last_restart = Time.now()
       @restart_required = false
     end
 
@@ -791,5 +850,43 @@ module Synapse
 
       return name
     end
+
+    def update_state_file(watchers)
+      log.info "synapse: writing state file"
+
+      timestamp = Time.now.to_i
+
+      # Remove stale backends
+      @seen.each do |watcher_name, backends|
+        backends.each do |backend_name, backend|
+          ts = backend.fetch('timestamp', 0)
+          delta = (timestamp - ts).abs
+          if delta > @state_file_ttl
+            log.info "synapse: expiring #{backend_name} with age #{delta}"
+            backends.delete(backend_name)
+          end
+        end
+      end
+
+      # Remove any services which no longer have any backends
+      @seen = @seen.reject{|watcher_name, backends| backends.keys.length == 0}
+
+      # Add backends from watchers
+      watchers.each do |watcher|
+        unless @seen.key?(watcher.name)
+          @seen[watcher.name] = {}
+        end
+
+        watcher.backends.each do |backend|
+          backend_name = construct_name(backend)
+          @seen[watcher.name][backend_name] = backend.merge('timestamp' => timestamp)
+        end
+      end
+
+      # Atomically write new state file
+      tmp_state_file_path = @state_file_path + ".tmp"
+      File.write(tmp_state_file_path, JSON.pretty_generate(@seen))
+      FileUtils.mv(tmp_state_file_path, @state_file_path)
+    end
   end
 end

data/lib/synapse/service_watcher/base.rb

@@ -1,3 +1,4 @@
+require 'set'
 require 'synapse/log'
 
 module Synapse
@@ -42,6 +43,10 @@ module Synapse
 
       @keep_default_servers = opts['keep_default_servers'] || false
 
+      # If there are no default servers and a watcher reports no backends, then
+      # use the previous backends that we already know about.
+      @use_previous_backends = opts.fetch('use_previous_backends', true)
+
       # set a flag used to tell the watchers to exit
       # this is not used in every watcher
       @should_exit = false
@@ -95,13 +100,46 @@ module Synapse
     end
 
     def set_backends(new_backends)
-      if @keep_default_servers
-        @backends = @default_servers + new_backends
+      # Aggregate and deduplicate all potential backend service instances.
+      new_backends = (new_backends + @default_servers) if @keep_default_servers
+      new_backends = new_backends.uniq {|b|
+        [b['host'], b['port'], b.fetch('name', '')]
+      }
+
+      if new_backends.to_set == @backends.to_set
+        return false
+      end
+
+      if new_backends.empty?
+        if @default_servers.empty?
+          if @use_previous_backends
+            # Discard this update
+            log.warn "synapse: no backends for service #{@name} and no default" \
+              " servers for service #{@name}; using previous backends: #{@backends.inspect}"
+            return false
+          else
+            log.warn "synapse: no backends for service #{@name}, no default" \
+              " servers for service #{@name} and 'use_previous_backends' is disabled;" \
+              " dropping all backends"
+            @backends.clear
+          end
+        else
+          log.warn "synapse: no backends for service #{@name};" \
+            " using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
       else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
         @backends = new_backends
       end
+
+      reconfigure!
+
+      return true
     end
 
+    # Subclasses should not invoke this directly; it's only exposed so that it
+    # can be overridden in subclasses.
     def reconfigure!
       @synapse.reconfigure!
     end

data/lib/synapse/service_watcher/dns.rb

@@ -89,21 +89,7 @@ module Synapse
         end
       end
 
-      if new_backends.empty?
-        if @default_servers.empty?
-          log.warn "synapse: no backends and no default servers for service #{@name};" \
-            " using previous backends: #{@backends.inspect}"
-        else
-          log.warn "synapse: no backends for service #{@name};" \
-            " using default servers: #{@default_servers.inspect}"
-          @backends = @default_servers
-        end
-      else
-        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
-        set_backends(new_backends)
-      end
-
-      reconfigure!
+      set_backends(new_backends)
     end
   end
 end

data/lib/synapse/service_watcher/docker.rb

@@ -23,16 +23,10 @@ module Synapse
     end
 
     def watch
-      last_containers = []
       until @should_exit
         begin
           start = Time.now
-          current_containers = containers
-          unless last_containers == current_containers
-            last_containers = current_containers
-            configure_backends(last_containers)
-          end
-
+          set_backends(containers)
           sleep_until_next_check(start)
         rescue Exception => e
           log.warn "synapse: error in watcher thread: #{e.inspect}"
@@ -98,23 +92,5 @@ module Synapse
       log.warn "synapse: error while polling for containers: #{e.inspect}"
       []
     end
-
-    def configure_backends(new_backends)
-      if new_backends.empty?
-        if @default_servers.empty?
-          log.warn "synapse: no backends and no default servers for service #{@name};" \
-            " using previous backends: #{@backends.inspect}"
-        else
-          log.warn "synapse: no backends for service #{@name};" \
-            " using default servers: #{@default_servers.inspect}"
-          @backends = @default_servers
-        end
-      else
-        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
-        set_backends(new_backends)
-      end
-      reconfigure!
-    end
-
   end
 end

data/lib/synapse/service_watcher/ec2tag.rb

@@ -1,25 +1,107 @@
-require "synapse/service_watcher/base"
+require 'synapse/service_watcher/base'
+require 'aws-sdk'
 
 module Synapse
   class EC2Watcher < BaseWatcher
+
+    attr_reader :check_interval
+
     def start
-      # connect to ec2
-      # find all servers whose @discovery['tag_name'] matches @discovery['tag_value']
-      # call @synapse.configure
+      region = @discovery['aws_region'] || ENV['AWS_REGION']
+      log.info "Connecting to EC2 region: #{region}"
+
+      @ec2 = AWS::EC2.new(
+        region:            region,
+        access_key_id:     @discovery['aws_access_key_id']     || ENV['AWS_ACCESS_KEY_ID'],
+        secret_access_key: @discovery['aws_secret_access_key'] || ENV['AWS_SECRET_ACCESS_KEY'] )
+
+      @check_interval = @discovery['check_interval'] || 15.0
+
+      log.info "synapse: ec2tag watcher looking for instances " +
+        "tagged with #{@discovery['tag_name']}=#{@discovery['tag_value']}"
+
+      @watcher = Thread.new { watch }
     end
 
     private
+
     def validate_discovery_opts
+      # Required, via options only.
       raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
-        unless @discovery['method'] == 'ec2tag' 
-      raise ArgumentError, "a `server_port_override` option is required for ec2tag watchers" \
-        unless @server_port_override
-      raise ArgumentError, "missing aws credentials for service #{@name}" \
-        unless (@discovery['aws_key'] && @discovery['aws_secret'])
+        unless @discovery['method'] == 'ec2tag'
       raise ArgumentError, "aws tag name is required for service #{@name}" \
         unless @discovery['tag_name']
       raise ArgumentError, "aws tag value required for service #{@name}" \
         unless @discovery['tag_value']
+
+      # As we're only looking up instances with hostnames/IPs, need to
+      # be explicitly told which port the service we're balancing for listens on.
+      unless @haproxy['server_port_override']
+        raise ArgumentError,
+          "Missing server_port_override for service #{@name} - which port are backends listening on?"
+      end
+
+      unless @haproxy['server_port_override'].match(/^\d+$/)
+        raise ArgumentError, "Invalid server_port_override value"
+      end
+
+      # Required, but can use well-known environment variables.
+      %w[aws_access_key_id aws_secret_access_key aws_region].each do |attr|
+        unless (@discovery[attr] || ENV[attr.upcase])
+          raise ArgumentError, "Missing #{attr} option or #{attr.upcase} environment variable"
+        end
+      end
+    end
+
+    def watch
+      until @should_exit
+        begin
+          start = Time.now
+          if set_backends(discover_instances)
+            log.info "synapse: ec2tag watcher backends have changed."
+          end
+          sleep_until_next_check(start)
+        rescue Exception => e
+          log.warn "synapse: error in ec2tag watcher thread: #{e.inspect}"
+          log.warn e.backtrace
+        end
+      end
+
+      log.info "synapse: ec2tag watcher exited successfully"
+    end
+
+    def sleep_until_next_check(start_time)
+      sleep_time = check_interval - (Time.now - start_time)
+      if sleep_time > 0.0
+        sleep(sleep_time)
+      end
+    end
+
+    def discover_instances
+      AWS.memoize do
+        instances = instances_with_tags(@discovery['tag_name'], @discovery['tag_value'])
+
+        new_backends = []
+
+        # choice of private_dns_name, dns_name, private_ip_address or
+        # ip_address, for now, just stick with the private fields.
+        instances.each do |instance|
+          new_backends << {
+            'name' => instance.private_dns_name,
+            'host' => instance.private_ip_address,
+            'port' => @haproxy['server_port_override'],
+          }
+        end
+
+        new_backends
+      end
+    end
+
+    def instances_with_tags(tag_name, tag_value)
+      @ec2.instances
+        .tagged(tag_name)
+        .tagged_values(tag_value)
+        .select { |i| i.status == :running }
     end
   end
 end