synapse 0.13.8 → 0.14.0

data/lib/synapse/service_watcher/README.md

@@ -1,12 +1,13 @@
 ## Watcher Classes
 
 Watchers are the piece of Synapse that watch an external service registry
-and reflect those changes in the local HAProxy state. Watchers should conform
-to the interface specified by `BaseWatcher` and when your watcher has received
-an update from the service registry you should call
+and reflect those changes in the local configuration state. Watchers should
+conform to the interface specified by `BaseWatcher` and when your watcher has
+received an update from the service registry you should call
 `set_backends(new_backends)` to trigger a sync of your watcher state with local
-HAProxy state. See the [`Backend Interface`](#backend_interface) section for
-what service registrations Synapse understands.
+configuration (HAProxy, files, etc ...) state. See the
+[`Backend Interface`](#backend_interface) section for what fields in
+registrations Synapse understands.
 
 ```ruby
 require "synapse/service_watcher/base"

data/lib/synapse/service_watcher/base.rb

@@ -7,7 +7,7 @@ class Synapse::ServiceWatcher
 
     LEADER_WARN_INTERVAL = 30
 
-    attr_reader :name, :haproxy
+    attr_reader :name, :config_for_generator
 
     def initialize(opts={}, synapse)
       super()
@@ -15,7 +15,7 @@ class Synapse::ServiceWatcher
       @synapse = synapse
 
       # set required service parameters
-      %w{name discovery haproxy}.each do |req|
+      %w{name discovery}.each do |req|
         raise ArgumentError, "missing required option #{req}" unless opts[req]
       end
 
@@ -33,17 +33,15 @@ class Synapse::ServiceWatcher
       @leader_election = opts['leader_election'] || false
       @leader_last_warn = Time.now - LEADER_WARN_INTERVAL
 
-      # the haproxy config
-      @haproxy = opts['haproxy']
-      @haproxy['server_options'] ||= ""
-      @haproxy['server_port_override'] ||= nil
-      %w{backend frontend listen}.each do |sec|
-        @haproxy[sec] ||= []
-      end
-
-      unless @haproxy.include?('port')
-        log.warn "synapse: service #{name}: haproxy config does not include a port; only backend sections for the service will be created; you must move traffic there manually using configuration in `extra_sections`"
-      end
+      @config_for_generator = Hash[
+        @synapse.available_generators.collect do |generator_name, generator|
+          watcher_provided_config = opts[generator_name] || {}
+          normalized_generator_opts = generator.normalize_watcher_provided_config(
+            @name, watcher_provided_config
+          )
+          [generator_name, normalized_generator_opts]
+        end
+      ]
 
       # set initial backends to default servers, if any
       @default_servers = opts['default_servers'] || []
@@ -55,6 +53,22 @@ class Synapse::ServiceWatcher
       # use the previous backends that we already know about.
       @use_previous_backends = opts.fetch('use_previous_backends', true)
 
+      # If backends discovered with this watcher lack ports or discovered
+      # ports should be ignored, this option indicates that
+      @backend_port_override = opts['backend_port_override'] || nil
+
+      # For backwards compatability we support server_port_override
+      # This will be removed in future versions
+      if @backend_port_override.nil? && @config_for_generator['haproxy']
+        @backend_port_override = @config_for_generator['haproxy']['server_port_override']
+      end
+
+      unless @backend_port_override.nil?
+        unless @backend_port_override.to_s.match(/^\d+$/)
+          raise ArgumentError, "Invalid backend_port_override value"
+        end
+      end
+
       # set a flag used to tell the watchers to exit
       # this is not used in every watcher
       @should_exit = false
@@ -62,6 +76,11 @@ class Synapse::ServiceWatcher
       validate_discovery_opts
     end
 
+    def haproxy
+      log.warn "synapse: service watcher #{@name} accessing watcher.haproxy. This is DEPRECATED and will be removed in future iterations, use watcher.config_for_generator['haproxy'] instead."
+      config_for_generator['haproxy']
+    end
+
     # this should be overridden to actually start your watcher
     def start
       log.info "synapse: starting stub watcher; this means doing nothing at all!"
@@ -135,6 +154,12 @@ class Synapse::ServiceWatcher
     def set_backends(new_backends)
       # Aggregate and deduplicate all potential backend service instances.
       new_backends = (new_backends + @default_servers) if @keep_default_servers
+      # Substitute backend_port_override for the provided port
+      new_backends = new_backends.collect do |b|
+        port = @backend_port_override ? @backend_port_override : b['port']
+        b.merge({'port' => port})
+      end
+
       new_backends = new_backends.uniq {|b|
         [b['host'], b['port'], b.fetch('name', '')]
       }

data/lib/synapse/service_watcher/ec2tag.rb

@@ -36,13 +36,9 @@ class Synapse::ServiceWatcher
 
       # As we're only looking up instances with hostnames/IPs, need to
       # be explicitly told which port the service we're balancing for listens on.
-      unless @haproxy['server_port_override']
+      unless @backend_port_override
         raise ArgumentError,
-          "Missing server_port_override for service #{@name} - which port are backends listening on?"
-      end
-
-      unless @haproxy['server_port_override'].to_s.match(/^\d+$/)
-        raise ArgumentError, "Invalid server_port_override value"
+          "Missing backend_port_override for service #{@name} - which port are backends listening on?"
       end
 
       # aws region is optional in the SDK, aws will use a default value if not provided
@@ -93,7 +89,6 @@ class Synapse::ServiceWatcher
           new_backends << {
             'name' => instance.private_dns_name,
             'host' => instance.private_ip_address,
-            'port' => @haproxy['server_port_override'],
           }
         end
 

data/lib/synapse/service_watcher/zookeeper.rb

@@ -143,15 +143,13 @@ class Synapse::ServiceWatcher
         rescue StandardError => e
           log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
         else
-          server_port = @haproxy['server_port_override'] ? @haproxy['server_port_override'] : port
-
           # find the numberic id in the node name; used for leader elections if enabled
           numeric_id = id.split('_').last
           numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
 
-          log.debug "synapse: discovered backend #{name} at #{host}:#{server_port} for service #{@name}"
+          log.debug "synapse: discovered backend #{name} at #{host}:#{port} for service #{@name}"
           new_backends << {
-            'name' => name, 'host' => host, 'port' => server_port,
+            'name' => name, 'host' => host, 'port' => port,
             'id' => numeric_id, 'weight' => weight,
             'haproxy_server_options' => haproxy_server_options,
             'labels' => labels

data/lib/synapse/service_watcher/zookeeper_dns.rb

@@ -218,7 +218,6 @@ class Synapse::ServiceWatcher
     def mk_child_watcher_opts(discovery_opts)
       {
         'name' => @name,
-        'haproxy' => @haproxy,
         'discovery' => discovery_opts,
         'default_servers' => @default_servers,
       }

data/lib/synapse/version.rb

@@ -1,3 +1,3 @@
 module Synapse
-  VERSION = "0.13.8"
+  VERSION = "0.14.0"
 end

data/spec/lib/synapse/file_output_spec.rb

@@ -1,8 +1,9 @@
 require 'spec_helper'
 require 'fileutils'
+require 'synapse/config_generator/file_output'
 
-describe Synapse::FileOutput do
-  subject { Synapse::FileOutput.new(config['file_output']) }
+describe Synapse::ConfigGenerator::FileOutput do
+  subject { Synapse::ConfigGenerator::FileOutput.new(config['file_output']) }
 
   before(:example) do
     FileUtils.mkdir_p(config['file_output']['output_directory'])

data/spec/lib/synapse/haproxy_spec.rb

@@ -1,16 +1,19 @@
 require 'spec_helper'
+require 'synapse/config_generator/haproxy'
 
 class MockWatcher; end;
 
-describe Synapse::Haproxy do
-  subject { Synapse::Haproxy.new(config['haproxy']) }
+describe Synapse::ConfigGenerator::Haproxy do
+  subject { Synapse::ConfigGenerator::Haproxy.new(config['haproxy']) }
 
   let(:mockwatcher) do
     mockWatcher = double(Synapse::ServiceWatcher)
     allow(mockWatcher).to receive(:name).and_return('example_service')
     backends = [{ 'host' => 'somehost', 'port' => 5555}]
     allow(mockWatcher).to receive(:backends).and_return(backends)
-    allow(mockWatcher).to receive(:haproxy).and_return({'server_options' => "check inter 2000 rise 3 fall 2"})
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'server_options' => "check inter 2000 rise 3 fall 2"}
+    })
     mockWatcher
   end
 
@@ -19,7 +22,9 @@ describe Synapse::Haproxy do
     allow(mockWatcher).to receive(:name).and_return('example_service2')
     backends = [{ 'host' => 'somehost', 'port' => 5555, 'haproxy_server_options' => 'backup'}]
     allow(mockWatcher).to receive(:backends).and_return(backends)
-    allow(mockWatcher).to receive(:haproxy).and_return({'server_options' => "check inter 2000 rise 3 fall 2"})
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'server_options' => "check inter 2000 rise 3 fall 2"}
+    })
     mockWatcher
   end
 
@@ -28,30 +33,174 @@ describe Synapse::Haproxy do
     allow(mockWatcher).to receive(:name).and_return('example_service3')
     backends = [{ 'host' => 'somehost', 'port' => 5555}]
     allow(mockWatcher).to receive(:backends).and_return(backends)
-    allow(mockWatcher).to receive(:haproxy).and_return({'server_options' => "check inter 2000 rise 3 fall 2", 'cookie_value_method' => 'hash'})
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'server_options' => "check inter 2000 rise 3 fall 2", 'cookie_value_method' => 'hash'}
+    })
     mockWatcher
   end
 
   let(:mockwatcher_frontend) do
     mockWatcher = double(Synapse::ServiceWatcher)
     allow(mockWatcher).to receive(:name).and_return('example_service4')
-    allow(mockWatcher).to receive(:haproxy).and_return('port' => 2200)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'port' => 2200}
+    })
+    mockWatcher
+  end
+
+  let(:mockwatcher_frontend_with_bind_options) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('example_service4')
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {
+        'port' => 2200,
+      'bind_options' => 'ssl no-sslv3 crt /path/to/cert/example.pem ciphers ECDHE-ECDSA-CHACHA20-POLY1305'
+      }
+    })
     mockWatcher
   end
 
   let(:mockwatcher_frontend_with_bind_address) do
     mockWatcher = double(Synapse::ServiceWatcher)
     allow(mockWatcher).to receive(:name).and_return('example_service5')
-    allow(mockWatcher).to receive(:haproxy).and_return('port' => 2200, 'bind_address' => "127.0.0.3")
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'port' => 2200, 'bind_address' => "127.0.0.3"}
+    })
     mockWatcher
   end
 
+  let(:mockwatcher_disabled) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('disabled_watcher')
+    backends = [{ 'host' => 'somehost', 'port' => 5555}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'port' => 2200, 'disabled' => true}
+    })
+    mockWatcher
+  end
+
+  describe '#initialize' do
+    it 'succeeds on minimal config' do
+      conf = {
+        'global' => [],
+        'defaults' => [],
+        'do_writes' => false,
+        'do_reloads' => false,
+        'do_socket' => false
+      }
+      Synapse::ConfigGenerator::Haproxy.new(conf)
+      expect{Synapse::ConfigGenerator::Haproxy.new(conf)}.not_to raise_error
+    end
+
+    it 'validates req_pairs' do
+      req_pairs = {
+        'do_writes' => 'config_file_path',
+        'do_socket' => 'socket_file_path',
+        'do_reloads' => 'reload_command'
+      }
+      valid_conf = {
+        'global' => [],
+        'defaults' => [],
+        'do_reloads' => false,
+        'do_socket' => false,
+        'do_writes' => false
+      }
+
+      req_pairs.each do |key, value|
+        conf = valid_conf.clone
+        conf[key] = true
+        expect{Synapse::ConfigGenerator::Haproxy.new(conf)}.
+          to raise_error(ArgumentError, "the `#{value}` option is required when `#{key}` is true")
+      end
+
+    end
+
+    it 'properly defaults do_writes, do_socket, do_reloads' do
+      conf = {
+        'global' => [],
+        'defaults' => [],
+        'config_file_path' => 'test_file',
+        'socket_file_path' => 'test_socket',
+        'reload_command' => 'test_reload'
+      }
+
+      expect{Synapse::ConfigGenerator::Haproxy.new(conf)}.not_to raise_error
+      haproxy = Synapse::ConfigGenerator::Haproxy.new(conf)
+      expect(haproxy.opts['do_writes']).to eql(true)
+      expect(haproxy.opts['do_socket']).to eql(true)
+      expect(haproxy.opts['do_reloads']).to eql(true)
+    end
+
+    it 'complains when req_pairs are not passed at all' do
+      conf = {
+        'global' => [],
+        'defaults' => [],
+      }
+      expect{Synapse::ConfigGenerator::Haproxy.new(conf)}.to raise_error(ArgumentError)
+    end
+  end
+
   describe '#name' do
     it 'returns haproxy' do
       expect(subject.name).to eq('haproxy')
     end
   end
 
+  describe 'disabled watcher' do
+    let(:watchers) { [mockwatcher, mockwatcher_disabled] }
+    let(:socket_file_path) { 'socket_file_path' }
+
+    before do
+      config['haproxy']['do_socket'] = true
+      config['haproxy']['socket_file_path'] = socket_file_path
+    end
+
+    it 'does not generate config' do
+      allow(subject).to receive(:parse_watcher_config).and_return({})
+      expect(subject).to receive(:generate_frontend_stanza).exactly(:once).with(mockwatcher, nil)
+      expect(subject).to receive(:generate_backend_stanza).exactly(:once).with(mockwatcher, nil)
+      subject.update_config(watchers)
+    end
+
+    context 'when configuration via the socket succeeds' do
+      before do
+        subject.instance_variable_set(:@restart_required, false)
+        allow(subject).to receive(:generate_config).exactly(:once).and_return 'mock_config'
+      end
+
+      it 'does not cause a restart due to the socket' do
+        mock_socket_output = "example_service,somehost:5555"
+        allow(subject).to receive(:talk_to_socket).with(socket_file_path, "show stat\n").and_return mock_socket_output
+
+        expect(subject).to receive(:talk_to_socket).exactly(:once).with(
+          socket_file_path, "enable server example_service/somehost:5555\n"
+        ).and_return "\n"
+
+        subject.update_config(watchers)
+
+        expect(subject.instance_variable_get(:@restart_required)).to eq false
+      end
+
+      it 'disables existing servers on the socket' do
+        mock_socket_output = "example_service,somehost:5555\ndisabled_watcher,somehost:5555"
+        allow(subject).to receive(:talk_to_socket).with(socket_file_path, "show stat\n").and_return mock_socket_output
+
+
+        expect(subject).to receive(:talk_to_socket).exactly(:once).with(
+          socket_file_path, "enable server example_service/somehost:5555\n"
+        ).and_return "\n"
+        expect(subject).to receive(:talk_to_socket).exactly(:once).with(
+          socket_file_path, "disable server disabled_watcher/somehost:5555\n"
+        ).and_return "\n"
+
+        subject.update_config(watchers)
+
+        expect(subject.instance_variable_get(:@restart_required)).to eq false
+      end
+    end
+  end
+
   describe '#update_config' do
     let(:watchers) { [mockwatcher_frontend, mockwatcher_frontend_with_bind_address] }
 
@@ -254,7 +403,12 @@ describe Synapse::Haproxy do
       context "when #{order_option} is specified for backend_order" do
         it 'generates backend stanza in correct order' do
           mockConfig = []
-          allow(mockwatcher_with_multiple_backends).to receive(:haproxy).and_return({'server_options' => "check inter 2000 rise 3 fall 2", 'backend_order' => order_option})
+          allow(mockwatcher_with_multiple_backends).to receive(:config_for_generator).and_return({
+            'haproxy' => {
+              'server_options' => "check inter 2000 rise 3 fall 2",
+              'backend_order' => order_option
+            }
+          })
           expect(subject.generate_backend_stanza(mockwatcher_with_multiple_backends, mockConfig)).to eql(multiple_backends_stanza_map[order_option])
         end
       end
@@ -281,6 +435,11 @@ describe Synapse::Haproxy do
     expect(subject.generate_frontend_stanza(mockwatcher_frontend, mockConfig)).to eql(["\nfrontend example_service4", [], "\tbind localhost:2200", "\tdefault_backend example_service4"])
   end
 
+  it 'generates frontend stanza with bind options ' do
+    mockConfig = []
+    expect(subject.generate_frontend_stanza(mockwatcher_frontend_with_bind_options, mockConfig)).to eql(["\nfrontend example_service4", [], "\tbind localhost:2200 ssl no-sslv3 crt /path/to/cert/example.pem ciphers ECDHE-ECDSA-CHACHA20-POLY1305", "\tdefault_backend example_service4"])
+  end
+
   it 'respects frontend bind_address ' do
     mockConfig = []
     expect(subject.generate_frontend_stanza(mockwatcher_frontend_with_bind_address, mockConfig)).to eql(["\nfrontend example_service5", [], "\tbind 127.0.0.3:2200", "\tdefault_backend example_service5"])

data/spec/lib/synapse/service_watcher_base_spec.rb

@@ -5,7 +5,14 @@ class Synapse::ServiceWatcher::BaseWatcher
 end
 
 describe Synapse::ServiceWatcher::BaseWatcher do
-  let(:mocksynapse) { double() }
+  let(:mocksynapse) do
+    mock_synapse = instance_double(Synapse::Synapse)
+    mockgenerator = Synapse::ConfigGenerator::BaseGenerator.new()
+    allow(mock_synapse).to receive(:available_generators).and_return({
+      'haproxy' => mockgenerator
+    })
+    mock_synapse
+  end
   subject { Synapse::ServiceWatcher::BaseWatcher.new(args, mocksynapse) }
   let(:testargs) { { 'name' => 'foo', 'discovery' => { 'method' => 'base' }, 'haproxy' => {} }}
 
@@ -20,7 +27,7 @@ describe Synapse::ServiceWatcher::BaseWatcher do
     it('can at least construct') { expect { subject }.not_to raise_error }
   end
 
-  ['name', 'discovery', 'haproxy'].each do |to_remove|
+  ['name', 'discovery'].each do |to_remove|
     context "without #{to_remove} argument" do
       let(:args) { remove_arg to_remove }
       it('gots bang') { expect { subject }.to raise_error(ArgumentError, "missing required option #{to_remove}") }

data/spec/lib/synapse/service_watcher_docker_spec.rb

@@ -7,7 +7,14 @@ class Synapse::ServiceWatcher::DockerWatcher
 end
 
 describe Synapse::ServiceWatcher::DockerWatcher do
-  let(:mocksynapse) { double() }
+  let(:mocksynapse) do
+    mock_synapse = instance_double(Synapse::Synapse)
+    mockgenerator = Synapse::ConfigGenerator::BaseGenerator.new()
+    allow(mock_synapse).to receive(:available_generators).and_return({
+      'haproxy' => mockgenerator
+    })
+    mock_synapse
+  end
   subject { Synapse::ServiceWatcher::DockerWatcher.new(testargs, mocksynapse) }
   let(:testargs) { { 'name' => 'foo', 'discovery' => { 'method' => 'docker', 'servers' => [{'host' => 'server1.local', 'name' => 'mainserver'}], 'image_name' => 'mycool/image', 'container_port' => 6379 }, 'haproxy' => {} }}
   before(:each) do