synapse 0.15.1 → 0.16.2

data/lib/synapse/statsd.rb

@@ -0,0 +1,47 @@
+require 'datadog/statsd'
+require 'synapse/log'
+
+module Synapse
+  module StatsD
+    def statsd
+      @@STATSD ||= StatsD.statsd_for(self.class.name)
+    end
+
+    def statsd_increment(key, tags = [])
+      statsd.increment(key, tags: tags, sample_rate: sample_rate_for(key))
+    end
+
+    def statsd_time(key, tags = [])
+      statsd.time(key, tags: tags, sample_rate: sample_rate_for(key)) do
+        yield
+      end
+    end
+
+    class << self
+      include Logging
+
+      @@STATSD_HOST = "localhost"
+      @@STATSD_PORT = 8125
+      @@STATSD_SAMPLE_RATE = {}
+
+      def statsd_for(classname)
+        log.debug "synapse: creating statsd client for class '#{classname}' on host '#{@@STATSD_HOST}' port #{@@STATSD_PORT}"
+        Datadog::Statsd.new(@@STATSD_HOST, @@STATSD_PORT)
+      end
+
+      def configure_statsd(opts)
+        @@STATSD_HOST = opts['host'] || @@STATSD_HOST
+        @@STATSD_PORT = (opts['port'] || @@STATSD_PORT).to_i
+        @@STATSD_SAMPLE_RATE = opts['sample_rate'] || {}
+        log.info "synapse: configuring statsd on host '#{@@STATSD_HOST}' port #{@@STATSD_PORT}"
+      end
+    end
+
+    private
+
+    def sample_rate_for(key)
+      rate = @@STATSD_SAMPLE_RATE[key]
+      rate.nil? ? 1 : rate
+    end
+  end
+end

data/lib/synapse/version.rb

@@ -1,3 +1,3 @@
 module Synapse
-  VERSION = "0.15.1"
+  VERSION = "0.16.2"
 end

data/spec/bin/synapse_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+require 'tempfile'
+
+describe 'parseconfig' do
+  it 'parses a templated config' do
+    allow_any_instance_of(Synapse::Synapse).to receive(:run)
+    expect(Synapse::Synapse).to receive(:new).
+      with(hash_including(
+       {"services" =>
+          {"test" =>
+            {"default_servers" =>
+              [{"name" => "default1", "host" => "localhost", "port" => 8080}],
+             "discovery" =>
+              {"method" => "zookeeper",
+               "path" => "/airbnb/service/logging/event_collector",
+               "hosts" => ["localhost:2181"],
+               "label_filters" =>
+                [{"label" => "tag", "value" => "config value", "condition" => "equals"}]},
+             "haproxy" =>
+              {"port" => 3219,
+               "bind_address" => "localhost",
+               "server_options" => ["some_haproxy_server_option"],
+               "listen" => ["some_haproxy_listen_option"]}}},
+         "haproxy" =>
+          {"reload_command" => "sudo service haproxy reload",
+           "config_file_path" => "/etc/haproxy/haproxy.cfg",
+           "do_writes" => false,
+           "do_reloads" => false,
+           "do_socket" => false,
+           "global" => ["global_test_option"],
+           "defaults" => ["default_test_option"]},
+         "file_output" => {"output_directory" => "/tmp/synapse_file_output_test"}}
+      )).and_call_original
+    stub_const 'ENV', ENV.to_hash.merge(
+      {"SYNAPSE_CONFIG" => "#{File.dirname(__FILE__)}/../support/minimum.conf.yaml",
+      "SYNAPSE_CONFIG_VALUE" => "config value"}
+    )
+    stub_const 'ARGV', []
+    load "#{File.dirname(__FILE__)}/../../bin/synapse"
+  end
+
+  it 'fails if templated config is invalid' do
+    allow_any_instance_of(Synapse::Synapse).to receive(:run)
+    tmpcfg = Tempfile.new 'synapse.conf.yaml'
+    tmpcfg.write '{:a => "<% if %>"}'
+    tmpcfg.flush
+    stub_const 'ENV', ENV.to_hash.merge(
+      {"SYNAPSE_CONFIG" => tmpcfg.to_path,
+      "SYNAPSE_CONFIG_VALUE" => "config value"}
+    )
+    stub_const 'ARGV', []
+    expect {
+      load "#{File.dirname(__FILE__)}/../../bin/synapse"
+    }.to raise_error(SyntaxError)
+  end
+end

data/spec/lib/synapse/file_output_spec.rb

@@ -36,6 +36,13 @@ describe Synapse::ConfigGenerator::FileOutput do
     mockWatcher
   end
 
+  let(:mockwatcher_with_no_name) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('no_name_service')
+    allow(mockWatcher).to receive(:config_for_generator).and_return({})
+    mockWatcher
+  end
+
   let(:mockwatcher_disabled) do
     mockWatcher = double(Synapse::ServiceWatcher)
     allow(mockWatcher).to receive(:name).and_return('disabled_service')
@@ -60,14 +67,14 @@ describe Synapse::ConfigGenerator::FileOutput do
   end
 
   it 'manages correct files' do
-    subject.update_config([mockwatcher_1, mockwatcher_2, mockwatcher_disabled])
+    subject.update_config([mockwatcher_1, mockwatcher_2, mockwatcher_disabled, mockwatcher_with_no_name])
     FileUtils.cd(config['file_output']['output_directory']) do
       expect(Dir.glob('*.json').sort).to eql(['example_service.json', 'foobar_service.json'])
     end
     # Should clean up after itself
     FileUtils.cd(config['file_output']['output_directory']) do
       FileUtils.touch('disabled_service.json')
-      subject.update_config([mockwatcher_1, mockwatcher_disabled])
+      subject.update_config([mockwatcher_1, mockwatcher_disabled, mockwatcher_with_no_name])
       expect(Dir.glob('*.json')).to eql(['example_service.json'])
     end
     # Should clean up after itself

data/spec/lib/synapse/haproxy_spec.rb

@@ -6,6 +6,12 @@ class MockWatcher; end;
 describe Synapse::ConfigGenerator::Haproxy do
   subject { Synapse::ConfigGenerator::Haproxy.new(config['haproxy']) }
 
+  let (:nerve_weights_subject) {
+    nerve_weights_subject = subject.clone
+    nerve_weights_subject.opts['use_nerve_weights'] = true
+    nerve_weights_subject
+  }
+
   let(:maxid) do
     Synapse::ConfigGenerator::Haproxy::MAX_SERVER_ID
   end
@@ -22,6 +28,29 @@ describe Synapse::ConfigGenerator::Haproxy do
     mockWatcher
   end
 
+  let(:mockwatcher_with_hashed_haproxy_server_options) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('example_service')
+    backends = [{ 'host' => 'somehost', 'port' => 5555, 'haproxy_server_options' => {'option_key' => 'option_value'}}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'server_options' => "check inter 2000 rise 3 fall 2"}
+    })
+    allow(mockWatcher).to receive(:revision).and_return(1)
+    mockWatcher
+  end
+
+  let(:mockwatcher_with_hashed_server_options) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('example_service2')
+    backends = [{ 'host' => 'somehost', 'port' => 5555, 'haproxy_server_options' => 'id 12 backup'}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'server_options' => {'hash_key' => 'check inter 2000 rise 3 fall 2'}}
+    })
+    mockWatcher
+  end
+
   let(:mockwatcher_with_server_options) do
     mockWatcher = double(Synapse::ServiceWatcher)
     allow(mockWatcher).to receive(:name).and_return('example_service2')
@@ -84,6 +113,54 @@ describe Synapse::ConfigGenerator::Haproxy do
     mockWatcher
   end
 
+  let(:mockwatcher_with_weight) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('example_weighted_service')
+    backends = [{ 'host' => 'somehost', 'port' => 5555, 'weight' => 1}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {
+      }
+    })
+    mockWatcher
+  end
+
+  let(:mockwatcher_with_weight_as_string) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('example_weighted_service')
+    backends = [{ 'host' => 'somehost', 'port' => 5555, 'weight' => '1'}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {
+      }
+    })
+    mockWatcher
+  end
+
+  let(:mockwatcher_with_weight_as_hash) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('example_weighted_service')
+    backends = [{ 'host' => 'somehost', 'port' => 5555, 'weight' => {}}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {
+      }
+    })
+    mockWatcher
+  end
+
+  let(:mockwatcher_with_haproxy_weight_and_nerve_weight) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('example_weighted_service')
+    backends = [{ 'host' => 'somehost', 'port' => 5555, 'weight' => 99, 'haproxy_server_options' => 'weight 50'}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {
+      }
+    })
+    mockWatcher
+  end
+
   let(:mockwatcher_frontend) do
     mockWatcher = double(Synapse::ServiceWatcher)
     allow(mockWatcher).to receive(:name).and_return('example_service4')
@@ -135,6 +212,17 @@ describe Synapse::ConfigGenerator::Haproxy do
     mockWatcher
   end
 
+  let(:mockwatcher_with_server_option_templates) do
+    mockWatcher = double(Synapse::ServiceWatcher)
+    allow(mockWatcher).to receive(:name).and_return('example_service7')
+    backends = [{ 'host' => 'somehost', 'port' => 5555, 'haproxy_server_options' => 'id 12 backup'}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:config_for_generator).and_return({
+      'haproxy' => {'server_options' => "check port %{port} inter 2000 rise 3 fall 2"}
+    })
+    mockWatcher
+  end
+
   describe '#initialize' do
     it 'succeeds on minimal config' do
       conf = {
@@ -148,6 +236,20 @@ describe Synapse::ConfigGenerator::Haproxy do
       expect{Synapse::ConfigGenerator::Haproxy.new(conf)}.not_to raise_error
     end
 
+    it 'reads use_nerve_weights in config' do
+      conf = {
+        'global' => [],
+        'defaults' => [],
+        'do_writes' => false,
+        'do_reloads' => false,
+        'do_socket' => false,
+        'use_nerve_weights' => true
+      }
+      expect{Synapse::ConfigGenerator::Haproxy.new(conf)}.not_to raise_error
+      haproxy = Synapse::ConfigGenerator::Haproxy.new(conf)
+      expect(haproxy.opts['use_nerve_weights']).to eql(true)
+    end
+
     it 'validates req_pairs' do
       req_pairs = {
         'do_writes' => 'config_file_path',
@@ -171,7 +273,7 @@ describe Synapse::ConfigGenerator::Haproxy do
 
     end
 
-    it 'properly defaults do_writes, do_socket, do_reloads' do
+    it 'properly defaults do_writes, do_socket, do_reloads, use_nerve_weights' do
       conf = {
         'global' => [],
         'defaults' => [],
@@ -185,6 +287,7 @@ describe Synapse::ConfigGenerator::Haproxy do
       expect(haproxy.opts['do_writes']).to eql(true)
       expect(haproxy.opts['do_socket']).to eql(true)
       expect(haproxy.opts['do_reloads']).to eql(true)
+      expect(haproxy.opts['use_nerve_weights']).to eql(nil)
     end
 
     it 'complains when req_pairs are not passed at all' do
@@ -311,6 +414,13 @@ describe Synapse::ConfigGenerator::Haproxy do
         expect(subject).to receive(:write_config).with(new_config)
         subject.update_config(watchers)
       end
+
+      it 'writes the new config to the file system' do
+        expect(File).to receive(:read).and_return(nil)
+        expect(File).to receive(:write)
+        expect(FileUtils).to receive(:mv)
+        subject.update_config(watchers)
+      end
     end
 
     context 'if we do not support config writes' do
@@ -481,6 +591,16 @@ describe Synapse::ConfigGenerator::Haproxy do
     expect(subject.generate_backend_stanza(mockwatcher, mockConfig)).to eql(["\nbackend example_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 check inter 2000 rise 3 fall 2"]])
   end
 
+  it 'ignores non-strings of haproxy_server_options' do
+    mockConfig = []
+    expect(subject.generate_backend_stanza(mockwatcher_with_hashed_haproxy_server_options, mockConfig)).to eql(["\nbackend example_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 check inter 2000 rise 3 fall 2"]])
+  end
+
+  it 'ignores non-strings of server_options' do
+    mockConfig = []
+    expect(subject.generate_backend_stanza(mockwatcher_with_hashed_server_options, mockConfig)).to eql(["\nbackend example_service2", [], ["\tserver somehost:5555 somehost:5555 cookie somehost:5555 id 12 backup"]])
+  end
+
   describe 'when known backend gets offline' do
     let(:mockStateCache) do
       mockCache = double(Synapse::ConfigGenerator::Haproxy::HaproxyState)
@@ -610,6 +730,11 @@ describe Synapse::ConfigGenerator::Haproxy do
     expect(subject.generate_backend_stanza(mockwatcher_with_server_options, mockConfig)).to eql(["\nbackend example_service2", [], ["\tserver somehost:5555 somehost:5555 cookie somehost:5555 check inter 2000 rise 3 fall 2 id 12 backup"]])
   end
 
+  it 'templates haproxy backend options' do
+    mockConfig = []
+    expect(subject.generate_backend_stanza(mockwatcher_with_server_option_templates, mockConfig)).to eql(["\nbackend example_service7", [], ["\tserver somehost:5555 somehost:5555 cookie somehost:5555 check port 5555 inter 2000 rise 3 fall 2 id 12 backup"]])
+  end
+
   it 'respects haproxy_server_id' do
     mockConfig = []
     expect(subject.generate_backend_stanza(mockwatcher_with_server_id, mockConfig)).to eql(
@@ -623,6 +748,38 @@ describe Synapse::ConfigGenerator::Haproxy do
     )
   end
 
+  describe '#use_nerve_weights' do
+    it 'respects weight as integer' do
+      mockConfig = []
+      expect(nerve_weights_subject.generate_backend_stanza(mockwatcher_with_weight, mockConfig)).to eql(
+        ["\nbackend example_weighted_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 weight 1"]]
+        )
+    end
+
+    it 'ignores weight if not valid' do
+      mockConfig = []
+      expect(nerve_weights_subject.generate_backend_stanza(mockwatcher_with_weight_as_hash, mockConfig)).to eql(
+        ["\nbackend example_weighted_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555"]]
+        )
+    end
+
+    it 'ignores haproxy_server_options weight with use_nerve_weights true' do
+      mockConfig = []
+      expect(nerve_weights_subject.generate_backend_stanza(mockwatcher_with_haproxy_weight_and_nerve_weight, mockConfig)).to eql(
+        ["\nbackend example_weighted_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 weight 99"]]
+        )
+    end
+
+    it 'ignores nerve weight with use_nerve_weights false' do
+      mockConfig = []
+      expect(subject.generate_backend_stanza(mockwatcher_with_haproxy_weight_and_nerve_weight, mockConfig)).to eql(
+        ["\nbackend example_weighted_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 weight 50"]]
+        )
+    end
+  end
+
+
+
   it 'generates frontend stanza ' do
     mockConfig = []
     expect(subject.generate_frontend_stanza(mockwatcher_frontend, mockConfig)).to eql(["\nfrontend example_service4", [], "\tbind localhost:2200", "\tdefault_backend example_service4"])

data/spec/lib/synapse/service_watcher_dns_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require 'synapse/service_watcher/dns'
+
+describe Synapse::ServiceWatcher::DnsWatcher do
+  let(:mock_synapse) do
+    mock_synapse = instance_double(Synapse::Synapse)
+    mockgenerator = Synapse::ConfigGenerator::BaseGenerator.new()
+    allow(mock_synapse).to receive(:available_generators).and_return({
+      'haproxy' => mockgenerator
+    })
+    mock_synapse
+  end
+
+  let(:discovery) do
+    {
+      'method' => 'dns',
+      'servers' => servers,
+      'generator_config_path' => 'disabled',
+    }
+  end
+
+  let(:config) do
+    {
+      'name' => 'test',
+      'haproxy' => {},
+      'discovery' => discovery,
+    }
+  end
+
+  let (:servers) do
+    [
+      {'name' => 'test1', 'host' => 'localhost'},
+      {'name' => 'test2', 'host' => '127.0.0.1'},
+      {'name' => 'test3', 'host' => '::1'},
+    ]
+  end
+
+  subject { Synapse::ServiceWatcher::DnsWatcher.new(config, mock_synapse) }
+
+  it 'only resolves hostnames' do
+    resolver = instance_double("Resolv::DNS")
+    allow(subject).to receive(:resolver).and_return(resolver)
+    expect(resolver).to receive(:getaddresses).with('localhost').
+      and_return([Resolv::IPv4.create('127.0.0.2')])
+    expect(subject.send(:resolve_servers)).to eql([
+      [{'name' => 'test1', 'host' => 'localhost'}, ['127.0.0.2']],
+      [{'name' => 'test2', 'host' => '127.0.0.1'}, ['127.0.0.1']],
+      [{'name' => 'test3', 'host' => '::1'}, ['::1']],
+    ])
+  end
+end

data/spec/lib/synapse/service_watcher_zookeeper_spec.rb

@@ -124,6 +124,51 @@ describe Synapse::ServiceWatcher::ZookeeperWatcher do
       expect(subject).to receive(:set_backends).with([],{})
       subject.send(:watcher_callback).call
     end
+
+    it 'responds fail to ping? when the client is not in any of the connected/connecting/associatin state' do
+      expect(mock_zk).to receive(:associating?).and_return(false)
+      expect(mock_zk).to receive(:connecting?).and_return(false)
+      expect(mock_zk).to receive(:connected?).and_return(false)
+
+      subject.instance_variable_set('@zk', mock_zk)
+      expect(subject.ping?).to be false
+    end
+
+    context "generator_config_path" do
+      let(:discovery) { { 'method' => 'zookeeper', 'hosts' => 'somehost', 'path' => 'some/path', 'generator_config_path' => generator_config_path } }
+      before :each do
+        expect(subject).to receive(:watch)
+        expect(subject).to receive(:discover).and_call_original
+        expect(mock_zk).to receive(:children).with('some/path', {:watch=>true}).and_return(
+          ["test_child_1"]
+        )
+        expect(mock_zk).to receive(:get).with('some/path/test_child_1').and_raise(ZK::Exceptions::NoNode)
+
+        subject.instance_variable_set('@zk', mock_zk)
+        expect(subject).to receive(:set_backends).with([],{})
+      end
+
+      context 'when generator_config_path is defined' do
+        let(:generator_config_path) { 'some/other/path' }
+        it 'reads from generator_config_path znode' do
+          expect(mock_zk).to receive(:get).with(generator_config_path, {:watch=>true}).and_return("")
+
+          subject.send(:watcher_callback).call
+        end
+
+        it 'does not crash if there is no zk node' do
+          expect(mock_zk).to receive(:get).with(generator_config_path, {:watch=>true}).and_raise(ZK::Exceptions::NoNode)
+
+          subject.send(:watcher_callback).call
+        end
+      end
+      context 'when generator_config_path is disabled' do
+        let(:generator_config_path) { 'disabled' }
+        it 'does not read from any znode' do
+            subject.send(:watcher_callback).call
+        end
+      end
+    end
   end
 
   context 'ZookeeperDnsWatcher' do