synapse 0.15.1 → 0.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,47 @@
1
+ require 'datadog/statsd'
2
+ require 'synapse/log'
3
+
4
+ module Synapse
5
+ module StatsD
6
+ def statsd
7
+ @@STATSD ||= StatsD.statsd_for(self.class.name)
8
+ end
9
+
10
+ def statsd_increment(key, tags = [])
11
+ statsd.increment(key, tags: tags, sample_rate: sample_rate_for(key))
12
+ end
13
+
14
+ def statsd_time(key, tags = [])
15
+ statsd.time(key, tags: tags, sample_rate: sample_rate_for(key)) do
16
+ yield
17
+ end
18
+ end
19
+
20
+ class << self
21
+ include Logging
22
+
23
+ @@STATSD_HOST = "localhost"
24
+ @@STATSD_PORT = 8125
25
+ @@STATSD_SAMPLE_RATE = {}
26
+
27
+ def statsd_for(classname)
28
+ log.debug "synapse: creating statsd client for class '#{classname}' on host '#{@@STATSD_HOST}' port #{@@STATSD_PORT}"
29
+ Datadog::Statsd.new(@@STATSD_HOST, @@STATSD_PORT)
30
+ end
31
+
32
+ def configure_statsd(opts)
33
+ @@STATSD_HOST = opts['host'] || @@STATSD_HOST
34
+ @@STATSD_PORT = (opts['port'] || @@STATSD_PORT).to_i
35
+ @@STATSD_SAMPLE_RATE = opts['sample_rate'] || {}
36
+ log.info "synapse: configuring statsd on host '#{@@STATSD_HOST}' port #{@@STATSD_PORT}"
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def sample_rate_for(key)
43
+ rate = @@STATSD_SAMPLE_RATE[key]
44
+ rate.nil? ? 1 : rate
45
+ end
46
+ end
47
+ end
@@ -1,3 +1,3 @@
1
1
  module Synapse
2
- VERSION = "0.15.1"
2
+ VERSION = "0.16.2"
3
3
  end
@@ -0,0 +1,56 @@
1
+ require 'spec_helper'
2
+ require 'tempfile'
3
+
4
+ describe 'parseconfig' do
5
+ it 'parses a templated config' do
6
+ allow_any_instance_of(Synapse::Synapse).to receive(:run)
7
+ expect(Synapse::Synapse).to receive(:new).
8
+ with(hash_including(
9
+ {"services" =>
10
+ {"test" =>
11
+ {"default_servers" =>
12
+ [{"name" => "default1", "host" => "localhost", "port" => 8080}],
13
+ "discovery" =>
14
+ {"method" => "zookeeper",
15
+ "path" => "/airbnb/service/logging/event_collector",
16
+ "hosts" => ["localhost:2181"],
17
+ "label_filters" =>
18
+ [{"label" => "tag", "value" => "config value", "condition" => "equals"}]},
19
+ "haproxy" =>
20
+ {"port" => 3219,
21
+ "bind_address" => "localhost",
22
+ "server_options" => ["some_haproxy_server_option"],
23
+ "listen" => ["some_haproxy_listen_option"]}}},
24
+ "haproxy" =>
25
+ {"reload_command" => "sudo service haproxy reload",
26
+ "config_file_path" => "/etc/haproxy/haproxy.cfg",
27
+ "do_writes" => false,
28
+ "do_reloads" => false,
29
+ "do_socket" => false,
30
+ "global" => ["global_test_option"],
31
+ "defaults" => ["default_test_option"]},
32
+ "file_output" => {"output_directory" => "/tmp/synapse_file_output_test"}}
33
+ )).and_call_original
34
+ stub_const 'ENV', ENV.to_hash.merge(
35
+ {"SYNAPSE_CONFIG" => "#{File.dirname(__FILE__)}/../support/minimum.conf.yaml",
36
+ "SYNAPSE_CONFIG_VALUE" => "config value"}
37
+ )
38
+ stub_const 'ARGV', []
39
+ load "#{File.dirname(__FILE__)}/../../bin/synapse"
40
+ end
41
+
42
+ it 'fails if templated config is invalid' do
43
+ allow_any_instance_of(Synapse::Synapse).to receive(:run)
44
+ tmpcfg = Tempfile.new 'synapse.conf.yaml'
45
+ tmpcfg.write '{:a => "<% if %>"}'
46
+ tmpcfg.flush
47
+ stub_const 'ENV', ENV.to_hash.merge(
48
+ {"SYNAPSE_CONFIG" => tmpcfg.to_path,
49
+ "SYNAPSE_CONFIG_VALUE" => "config value"}
50
+ )
51
+ stub_const 'ARGV', []
52
+ expect {
53
+ load "#{File.dirname(__FILE__)}/../../bin/synapse"
54
+ }.to raise_error(SyntaxError)
55
+ end
56
+ end
@@ -36,6 +36,13 @@ describe Synapse::ConfigGenerator::FileOutput do
36
36
  mockWatcher
37
37
  end
38
38
 
39
+ let(:mockwatcher_with_no_name) do
40
+ mockWatcher = double(Synapse::ServiceWatcher)
41
+ allow(mockWatcher).to receive(:name).and_return('no_name_service')
42
+ allow(mockWatcher).to receive(:config_for_generator).and_return({})
43
+ mockWatcher
44
+ end
45
+
39
46
  let(:mockwatcher_disabled) do
40
47
  mockWatcher = double(Synapse::ServiceWatcher)
41
48
  allow(mockWatcher).to receive(:name).and_return('disabled_service')
@@ -60,14 +67,14 @@ describe Synapse::ConfigGenerator::FileOutput do
60
67
  end
61
68
 
62
69
  it 'manages correct files' do
63
- subject.update_config([mockwatcher_1, mockwatcher_2, mockwatcher_disabled])
70
+ subject.update_config([mockwatcher_1, mockwatcher_2, mockwatcher_disabled, mockwatcher_with_no_name])
64
71
  FileUtils.cd(config['file_output']['output_directory']) do
65
72
  expect(Dir.glob('*.json').sort).to eql(['example_service.json', 'foobar_service.json'])
66
73
  end
67
74
  # Should clean up after itself
68
75
  FileUtils.cd(config['file_output']['output_directory']) do
69
76
  FileUtils.touch('disabled_service.json')
70
- subject.update_config([mockwatcher_1, mockwatcher_disabled])
77
+ subject.update_config([mockwatcher_1, mockwatcher_disabled, mockwatcher_with_no_name])
71
78
  expect(Dir.glob('*.json')).to eql(['example_service.json'])
72
79
  end
73
80
  # Should clean up after itself
@@ -6,6 +6,12 @@ class MockWatcher; end;
6
6
  describe Synapse::ConfigGenerator::Haproxy do
7
7
  subject { Synapse::ConfigGenerator::Haproxy.new(config['haproxy']) }
8
8
 
9
+ let (:nerve_weights_subject) {
10
+ nerve_weights_subject = subject.clone
11
+ nerve_weights_subject.opts['use_nerve_weights'] = true
12
+ nerve_weights_subject
13
+ }
14
+
9
15
  let(:maxid) do
10
16
  Synapse::ConfigGenerator::Haproxy::MAX_SERVER_ID
11
17
  end
@@ -22,6 +28,29 @@ describe Synapse::ConfigGenerator::Haproxy do
22
28
  mockWatcher
23
29
  end
24
30
 
31
+ let(:mockwatcher_with_hashed_haproxy_server_options) do
32
+ mockWatcher = double(Synapse::ServiceWatcher)
33
+ allow(mockWatcher).to receive(:name).and_return('example_service')
34
+ backends = [{ 'host' => 'somehost', 'port' => 5555, 'haproxy_server_options' => {'option_key' => 'option_value'}}]
35
+ allow(mockWatcher).to receive(:backends).and_return(backends)
36
+ allow(mockWatcher).to receive(:config_for_generator).and_return({
37
+ 'haproxy' => {'server_options' => "check inter 2000 rise 3 fall 2"}
38
+ })
39
+ allow(mockWatcher).to receive(:revision).and_return(1)
40
+ mockWatcher
41
+ end
42
+
43
+ let(:mockwatcher_with_hashed_server_options) do
44
+ mockWatcher = double(Synapse::ServiceWatcher)
45
+ allow(mockWatcher).to receive(:name).and_return('example_service2')
46
+ backends = [{ 'host' => 'somehost', 'port' => 5555, 'haproxy_server_options' => 'id 12 backup'}]
47
+ allow(mockWatcher).to receive(:backends).and_return(backends)
48
+ allow(mockWatcher).to receive(:config_for_generator).and_return({
49
+ 'haproxy' => {'server_options' => {'hash_key' => 'check inter 2000 rise 3 fall 2'}}
50
+ })
51
+ mockWatcher
52
+ end
53
+
25
54
  let(:mockwatcher_with_server_options) do
26
55
  mockWatcher = double(Synapse::ServiceWatcher)
27
56
  allow(mockWatcher).to receive(:name).and_return('example_service2')
@@ -84,6 +113,54 @@ describe Synapse::ConfigGenerator::Haproxy do
84
113
  mockWatcher
85
114
  end
86
115
 
116
+ let(:mockwatcher_with_weight) do
117
+ mockWatcher = double(Synapse::ServiceWatcher)
118
+ allow(mockWatcher).to receive(:name).and_return('example_weighted_service')
119
+ backends = [{ 'host' => 'somehost', 'port' => 5555, 'weight' => 1}]
120
+ allow(mockWatcher).to receive(:backends).and_return(backends)
121
+ allow(mockWatcher).to receive(:config_for_generator).and_return({
122
+ 'haproxy' => {
123
+ }
124
+ })
125
+ mockWatcher
126
+ end
127
+
128
+ let(:mockwatcher_with_weight_as_string) do
129
+ mockWatcher = double(Synapse::ServiceWatcher)
130
+ allow(mockWatcher).to receive(:name).and_return('example_weighted_service')
131
+ backends = [{ 'host' => 'somehost', 'port' => 5555, 'weight' => '1'}]
132
+ allow(mockWatcher).to receive(:backends).and_return(backends)
133
+ allow(mockWatcher).to receive(:config_for_generator).and_return({
134
+ 'haproxy' => {
135
+ }
136
+ })
137
+ mockWatcher
138
+ end
139
+
140
+ let(:mockwatcher_with_weight_as_hash) do
141
+ mockWatcher = double(Synapse::ServiceWatcher)
142
+ allow(mockWatcher).to receive(:name).and_return('example_weighted_service')
143
+ backends = [{ 'host' => 'somehost', 'port' => 5555, 'weight' => {}}]
144
+ allow(mockWatcher).to receive(:backends).and_return(backends)
145
+ allow(mockWatcher).to receive(:config_for_generator).and_return({
146
+ 'haproxy' => {
147
+ }
148
+ })
149
+ mockWatcher
150
+ end
151
+
152
+ let(:mockwatcher_with_haproxy_weight_and_nerve_weight) do
153
+ mockWatcher = double(Synapse::ServiceWatcher)
154
+ allow(mockWatcher).to receive(:name).and_return('example_weighted_service')
155
+ backends = [{ 'host' => 'somehost', 'port' => 5555, 'weight' => 99, 'haproxy_server_options' => 'weight 50'}]
156
+ allow(mockWatcher).to receive(:backends).and_return(backends)
157
+ allow(mockWatcher).to receive(:config_for_generator).and_return({
158
+ 'haproxy' => {
159
+ }
160
+ })
161
+ mockWatcher
162
+ end
163
+
87
164
  let(:mockwatcher_frontend) do
88
165
  mockWatcher = double(Synapse::ServiceWatcher)
89
166
  allow(mockWatcher).to receive(:name).and_return('example_service4')
@@ -135,6 +212,17 @@ describe Synapse::ConfigGenerator::Haproxy do
135
212
  mockWatcher
136
213
  end
137
214
 
215
+ let(:mockwatcher_with_server_option_templates) do
216
+ mockWatcher = double(Synapse::ServiceWatcher)
217
+ allow(mockWatcher).to receive(:name).and_return('example_service7')
218
+ backends = [{ 'host' => 'somehost', 'port' => 5555, 'haproxy_server_options' => 'id 12 backup'}]
219
+ allow(mockWatcher).to receive(:backends).and_return(backends)
220
+ allow(mockWatcher).to receive(:config_for_generator).and_return({
221
+ 'haproxy' => {'server_options' => "check port %{port} inter 2000 rise 3 fall 2"}
222
+ })
223
+ mockWatcher
224
+ end
225
+
138
226
  describe '#initialize' do
139
227
  it 'succeeds on minimal config' do
140
228
  conf = {
@@ -148,6 +236,20 @@ describe Synapse::ConfigGenerator::Haproxy do
148
236
  expect{Synapse::ConfigGenerator::Haproxy.new(conf)}.not_to raise_error
149
237
  end
150
238
 
239
+ it 'reads use_nerve_weights in config' do
240
+ conf = {
241
+ 'global' => [],
242
+ 'defaults' => [],
243
+ 'do_writes' => false,
244
+ 'do_reloads' => false,
245
+ 'do_socket' => false,
246
+ 'use_nerve_weights' => true
247
+ }
248
+ expect{Synapse::ConfigGenerator::Haproxy.new(conf)}.not_to raise_error
249
+ haproxy = Synapse::ConfigGenerator::Haproxy.new(conf)
250
+ expect(haproxy.opts['use_nerve_weights']).to eql(true)
251
+ end
252
+
151
253
  it 'validates req_pairs' do
152
254
  req_pairs = {
153
255
  'do_writes' => 'config_file_path',
@@ -171,7 +273,7 @@ describe Synapse::ConfigGenerator::Haproxy do
171
273
 
172
274
  end
173
275
 
174
- it 'properly defaults do_writes, do_socket, do_reloads' do
276
+ it 'properly defaults do_writes, do_socket, do_reloads, use_nerve_weights' do
175
277
  conf = {
176
278
  'global' => [],
177
279
  'defaults' => [],
@@ -185,6 +287,7 @@ describe Synapse::ConfigGenerator::Haproxy do
185
287
  expect(haproxy.opts['do_writes']).to eql(true)
186
288
  expect(haproxy.opts['do_socket']).to eql(true)
187
289
  expect(haproxy.opts['do_reloads']).to eql(true)
290
+ expect(haproxy.opts['use_nerve_weights']).to eql(nil)
188
291
  end
189
292
 
190
293
  it 'complains when req_pairs are not passed at all' do
@@ -311,6 +414,13 @@ describe Synapse::ConfigGenerator::Haproxy do
311
414
  expect(subject).to receive(:write_config).with(new_config)
312
415
  subject.update_config(watchers)
313
416
  end
417
+
418
+ it 'writes the new config to the file system' do
419
+ expect(File).to receive(:read).and_return(nil)
420
+ expect(File).to receive(:write)
421
+ expect(FileUtils).to receive(:mv)
422
+ subject.update_config(watchers)
423
+ end
314
424
  end
315
425
 
316
426
  context 'if we do not support config writes' do
@@ -481,6 +591,16 @@ describe Synapse::ConfigGenerator::Haproxy do
481
591
  expect(subject.generate_backend_stanza(mockwatcher, mockConfig)).to eql(["\nbackend example_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 check inter 2000 rise 3 fall 2"]])
482
592
  end
483
593
 
594
+ it 'ignores non-strings of haproxy_server_options' do
595
+ mockConfig = []
596
+ expect(subject.generate_backend_stanza(mockwatcher_with_hashed_haproxy_server_options, mockConfig)).to eql(["\nbackend example_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 check inter 2000 rise 3 fall 2"]])
597
+ end
598
+
599
+ it 'ignores non-strings of server_options' do
600
+ mockConfig = []
601
+ expect(subject.generate_backend_stanza(mockwatcher_with_hashed_server_options, mockConfig)).to eql(["\nbackend example_service2", [], ["\tserver somehost:5555 somehost:5555 cookie somehost:5555 id 12 backup"]])
602
+ end
603
+
484
604
  describe 'when known backend gets offline' do
485
605
  let(:mockStateCache) do
486
606
  mockCache = double(Synapse::ConfigGenerator::Haproxy::HaproxyState)
@@ -610,6 +730,11 @@ describe Synapse::ConfigGenerator::Haproxy do
610
730
  expect(subject.generate_backend_stanza(mockwatcher_with_server_options, mockConfig)).to eql(["\nbackend example_service2", [], ["\tserver somehost:5555 somehost:5555 cookie somehost:5555 check inter 2000 rise 3 fall 2 id 12 backup"]])
611
731
  end
612
732
 
733
+ it 'templates haproxy backend options' do
734
+ mockConfig = []
735
+ expect(subject.generate_backend_stanza(mockwatcher_with_server_option_templates, mockConfig)).to eql(["\nbackend example_service7", [], ["\tserver somehost:5555 somehost:5555 cookie somehost:5555 check port 5555 inter 2000 rise 3 fall 2 id 12 backup"]])
736
+ end
737
+
613
738
  it 'respects haproxy_server_id' do
614
739
  mockConfig = []
615
740
  expect(subject.generate_backend_stanza(mockwatcher_with_server_id, mockConfig)).to eql(
@@ -623,6 +748,38 @@ describe Synapse::ConfigGenerator::Haproxy do
623
748
  )
624
749
  end
625
750
 
751
+ describe '#use_nerve_weights' do
752
+ it 'respects weight as integer' do
753
+ mockConfig = []
754
+ expect(nerve_weights_subject.generate_backend_stanza(mockwatcher_with_weight, mockConfig)).to eql(
755
+ ["\nbackend example_weighted_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 weight 1"]]
756
+ )
757
+ end
758
+
759
+ it 'ignores weight if not valid' do
760
+ mockConfig = []
761
+ expect(nerve_weights_subject.generate_backend_stanza(mockwatcher_with_weight_as_hash, mockConfig)).to eql(
762
+ ["\nbackend example_weighted_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555"]]
763
+ )
764
+ end
765
+
766
+ it 'ignores haproxy_server_options weight with use_nerve_weights true' do
767
+ mockConfig = []
768
+ expect(nerve_weights_subject.generate_backend_stanza(mockwatcher_with_haproxy_weight_and_nerve_weight, mockConfig)).to eql(
769
+ ["\nbackend example_weighted_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 weight 99"]]
770
+ )
771
+ end
772
+
773
+ it 'ignores nerve weight with use_nerve_weights false' do
774
+ mockConfig = []
775
+ expect(subject.generate_backend_stanza(mockwatcher_with_haproxy_weight_and_nerve_weight, mockConfig)).to eql(
776
+ ["\nbackend example_weighted_service", [], ["\tserver somehost:5555 somehost:5555 id 1 cookie somehost:5555 weight 50"]]
777
+ )
778
+ end
779
+ end
780
+
781
+
782
+
626
783
  it 'generates frontend stanza ' do
627
784
  mockConfig = []
628
785
  expect(subject.generate_frontend_stanza(mockwatcher_frontend, mockConfig)).to eql(["\nfrontend example_service4", [], "\tbind localhost:2200", "\tdefault_backend example_service4"])
@@ -0,0 +1,51 @@
1
+ require 'spec_helper'
2
+ require 'synapse/service_watcher/dns'
3
+
4
+ describe Synapse::ServiceWatcher::DnsWatcher do
5
+ let(:mock_synapse) do
6
+ mock_synapse = instance_double(Synapse::Synapse)
7
+ mockgenerator = Synapse::ConfigGenerator::BaseGenerator.new()
8
+ allow(mock_synapse).to receive(:available_generators).and_return({
9
+ 'haproxy' => mockgenerator
10
+ })
11
+ mock_synapse
12
+ end
13
+
14
+ let(:discovery) do
15
+ {
16
+ 'method' => 'dns',
17
+ 'servers' => servers,
18
+ 'generator_config_path' => 'disabled',
19
+ }
20
+ end
21
+
22
+ let(:config) do
23
+ {
24
+ 'name' => 'test',
25
+ 'haproxy' => {},
26
+ 'discovery' => discovery,
27
+ }
28
+ end
29
+
30
+ let (:servers) do
31
+ [
32
+ {'name' => 'test1', 'host' => 'localhost'},
33
+ {'name' => 'test2', 'host' => '127.0.0.1'},
34
+ {'name' => 'test3', 'host' => '::1'},
35
+ ]
36
+ end
37
+
38
+ subject { Synapse::ServiceWatcher::DnsWatcher.new(config, mock_synapse) }
39
+
40
+ it 'only resolves hostnames' do
41
+ resolver = instance_double("Resolv::DNS")
42
+ allow(subject).to receive(:resolver).and_return(resolver)
43
+ expect(resolver).to receive(:getaddresses).with('localhost').
44
+ and_return([Resolv::IPv4.create('127.0.0.2')])
45
+ expect(subject.send(:resolve_servers)).to eql([
46
+ [{'name' => 'test1', 'host' => 'localhost'}, ['127.0.0.2']],
47
+ [{'name' => 'test2', 'host' => '127.0.0.1'}, ['127.0.0.1']],
48
+ [{'name' => 'test3', 'host' => '::1'}, ['::1']],
49
+ ])
50
+ end
51
+ end
@@ -124,6 +124,51 @@ describe Synapse::ServiceWatcher::ZookeeperWatcher do
124
124
  expect(subject).to receive(:set_backends).with([],{})
125
125
  subject.send(:watcher_callback).call
126
126
  end
127
+
128
+ it 'responds fail to ping? when the client is not in any of the connected/connecting/associatin state' do
129
+ expect(mock_zk).to receive(:associating?).and_return(false)
130
+ expect(mock_zk).to receive(:connecting?).and_return(false)
131
+ expect(mock_zk).to receive(:connected?).and_return(false)
132
+
133
+ subject.instance_variable_set('@zk', mock_zk)
134
+ expect(subject.ping?).to be false
135
+ end
136
+
137
+ context "generator_config_path" do
138
+ let(:discovery) { { 'method' => 'zookeeper', 'hosts' => 'somehost', 'path' => 'some/path', 'generator_config_path' => generator_config_path } }
139
+ before :each do
140
+ expect(subject).to receive(:watch)
141
+ expect(subject).to receive(:discover).and_call_original
142
+ expect(mock_zk).to receive(:children).with('some/path', {:watch=>true}).and_return(
143
+ ["test_child_1"]
144
+ )
145
+ expect(mock_zk).to receive(:get).with('some/path/test_child_1').and_raise(ZK::Exceptions::NoNode)
146
+
147
+ subject.instance_variable_set('@zk', mock_zk)
148
+ expect(subject).to receive(:set_backends).with([],{})
149
+ end
150
+
151
+ context 'when generator_config_path is defined' do
152
+ let(:generator_config_path) { 'some/other/path' }
153
+ it 'reads from generator_config_path znode' do
154
+ expect(mock_zk).to receive(:get).with(generator_config_path, {:watch=>true}).and_return("")
155
+
156
+ subject.send(:watcher_callback).call
157
+ end
158
+
159
+ it 'does not crash if there is no zk node' do
160
+ expect(mock_zk).to receive(:get).with(generator_config_path, {:watch=>true}).and_raise(ZK::Exceptions::NoNode)
161
+
162
+ subject.send(:watcher_callback).call
163
+ end
164
+ end
165
+ context 'when generator_config_path is disabled' do
166
+ let(:generator_config_path) { 'disabled' }
167
+ it 'does not read from any znode' do
168
+ subject.send(:watcher_callback).call
169
+ end
170
+ end
171
+ end
127
172
  end
128
173
 
129
174
  context 'ZookeeperDnsWatcher' do