synapse 0.11.1 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/.travis.yml +3 -1
- data/Gemfile.lock +32 -16
- data/README.md +120 -95
- data/lib/synapse.rb +20 -6
- data/lib/synapse/file_output.rb +57 -0
- data/lib/synapse/haproxy.rb +122 -25
- data/lib/synapse/service_watcher/base.rb +40 -2
- data/lib/synapse/service_watcher/dns.rb +1 -15
- data/lib/synapse/service_watcher/docker.rb +1 -25
- data/lib/synapse/service_watcher/ec2tag.rb +91 -9
- data/lib/synapse/service_watcher/zookeeper.rb +53 -24
- data/lib/synapse/version.rb +1 -1
- data/spec/lib/synapse/haproxy_spec.rb +22 -3
- data/spec/lib/synapse/service_watcher_base_spec.rb +67 -9
- data/spec/lib/synapse/service_watcher_docker_spec.rb +1 -33
- data/spec/lib/synapse/service_watcher_ec2tags_spec.rb +187 -0
- data/spec/spec_helper.rb +6 -1
- data/spec/support/configuration.rb +0 -2
- data/synapse.gemspec +3 -2
- metadata +29 -10
data/lib/synapse/haproxy.rb
CHANGED
@@ -1,10 +1,12 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'json'
|
1
3
|
require 'synapse/log'
|
2
4
|
require 'socket'
|
3
5
|
|
4
6
|
module Synapse
|
5
7
|
class Haproxy
|
6
8
|
include Logging
|
7
|
-
attr_reader :opts
|
9
|
+
attr_reader :opts, :name
|
8
10
|
|
9
11
|
# these come from the documentation for haproxy 1.5
|
10
12
|
# http://haproxy.1wt.eu/download/1.5/doc/configuration.txt
|
@@ -43,6 +45,7 @@ module Synapse
|
|
43
45
|
"option abortonclose",
|
44
46
|
"option accept-invalid-http-response",
|
45
47
|
"option allbackups",
|
48
|
+
"option allredisp",
|
46
49
|
"option checkcache",
|
47
50
|
"option forceclose",
|
48
51
|
"option forwardfor",
|
@@ -173,6 +176,7 @@ module Synapse
|
|
173
176
|
"option accept-invalid-http-request",
|
174
177
|
"option accept-invalid-http-response",
|
175
178
|
"option allbackups",
|
179
|
+
"option allredisp",
|
176
180
|
"option checkcache",
|
177
181
|
"option clitcpka",
|
178
182
|
"option contstats",
|
@@ -386,6 +390,7 @@ module Synapse
|
|
386
390
|
"option accept-invalid-http-request",
|
387
391
|
"option accept-invalid-http-response",
|
388
392
|
"option allbackups",
|
393
|
+
"option allredisp",
|
389
394
|
"option checkcache",
|
390
395
|
"option clitcpka",
|
391
396
|
"option contstats",
|
@@ -523,20 +528,53 @@ module Synapse
|
|
523
528
|
end
|
524
529
|
|
525
530
|
@opts = opts
|
531
|
+
@name = 'haproxy'
|
532
|
+
|
533
|
+
@opts['do_writes'] = true unless @opts.key?('do_writes')
|
534
|
+
@opts['do_socket'] = true unless @opts.key?('do_socket')
|
535
|
+
@opts['do_reloads'] = true unless @opts.key?('do_reloads')
|
526
536
|
|
527
537
|
# how to restart haproxy
|
528
|
-
@restart_interval = 2
|
538
|
+
@restart_interval = @opts.fetch('restart_interval', 2).to_i
|
539
|
+
@restart_jitter = @opts.fetch('restart_jitter', 0).to_f
|
529
540
|
@restart_required = true
|
530
|
-
|
541
|
+
|
542
|
+
# virtual clock bookkeeping for controlling how often haproxy restarts
|
543
|
+
@time = 0
|
544
|
+
@next_restart = @time
|
531
545
|
|
532
546
|
# a place to store the parsed haproxy config from each watcher
|
533
547
|
@watcher_configs = {}
|
548
|
+
|
549
|
+
@state_file_path = @opts['state_file_path']
|
550
|
+
@state_file_ttl = @opts.fetch('state_file_ttl', 60 * 60 * 24).to_i
|
551
|
+
@seen = {}
|
552
|
+
|
553
|
+
unless @state_file_path.nil?
|
554
|
+
begin
|
555
|
+
@seen = JSON.load(File.read(@state_file_path))
|
556
|
+
rescue StandardError => e
|
557
|
+
# It's ok if the state file doesn't exist
|
558
|
+
end
|
559
|
+
end
|
560
|
+
end
|
561
|
+
|
562
|
+
def tick(watchers)
|
563
|
+
if @time % 60 == 0 && !@state_file_path.nil?
|
564
|
+
update_state_file(watchers)
|
565
|
+
end
|
566
|
+
|
567
|
+
@time += 1
|
568
|
+
|
569
|
+
# We potentially have to restart if the restart was rate limited
|
570
|
+
# in the original call to update_config
|
571
|
+
restart if @opts['do_reloads'] && @restart_required
|
534
572
|
end
|
535
573
|
|
536
574
|
def update_config(watchers)
|
537
575
|
# if we support updating backends, try that whenever possible
|
538
576
|
if @opts['do_socket']
|
539
|
-
update_backends(watchers)
|
577
|
+
update_backends(watchers)
|
540
578
|
else
|
541
579
|
@restart_required = true
|
542
580
|
end
|
@@ -655,17 +693,35 @@ module Synapse
|
|
655
693
|
end
|
656
694
|
|
657
695
|
def generate_backend_stanza(watcher, config)
|
696
|
+
backends = {}
|
697
|
+
|
698
|
+
# The ordering here is important. First we add all the backends in the
|
699
|
+
# disabled state...
|
700
|
+
@seen.fetch(watcher.name, []).each do |backend_name, backend|
|
701
|
+
backends[backend_name] = backend.merge('enabled' => false)
|
702
|
+
end
|
703
|
+
|
704
|
+
# ... and then we overwite any backends that the watchers know about,
|
705
|
+
# setting the enabled state.
|
706
|
+
watcher.backends.each do |backend|
|
707
|
+
backend_name = construct_name(backend)
|
708
|
+
backends[backend_name] = backend.merge('enabled' => true)
|
709
|
+
end
|
710
|
+
|
658
711
|
if watcher.backends.empty?
|
659
|
-
log.
|
712
|
+
log.debug "synapse: no backends found for watcher #{watcher.name}"
|
660
713
|
end
|
661
714
|
|
662
715
|
stanza = [
|
663
716
|
"\nbackend #{watcher.name}",
|
664
717
|
config.map {|c| "\t#{c}"},
|
665
|
-
|
666
|
-
|
667
|
-
"\tserver #{backend_name} #{backend['host']}:#{backend['port']}
|
668
|
-
|
718
|
+
backends.keys.shuffle.map {|backend_name|
|
719
|
+
backend = backends[backend_name]
|
720
|
+
b = "\tserver #{backend_name} #{backend['host']}:#{backend['port']}"
|
721
|
+
b = "#{b} cookie #{backend_name}" unless config.include?('mode tcp')
|
722
|
+
b = "#{b} #{watcher.haproxy['server_options']}"
|
723
|
+
b = "#{b} disabled" unless backend['enabled']
|
724
|
+
b }
|
669
725
|
]
|
670
726
|
end
|
671
727
|
|
@@ -702,27 +758,26 @@ module Synapse
|
|
702
758
|
next if watcher.backends.empty?
|
703
759
|
|
704
760
|
unless cur_backends.include? watcher.name
|
705
|
-
log.
|
761
|
+
log.info "synapse: restart required because we added new section #{watcher.name}"
|
706
762
|
@restart_required = true
|
707
|
-
|
763
|
+
next
|
708
764
|
end
|
709
765
|
|
710
766
|
watcher.backends.each do |backend|
|
711
767
|
backend_name = construct_name(backend)
|
712
|
-
|
713
|
-
|
768
|
+
if cur_backends[watcher.name].include? backend_name
|
769
|
+
enabled_backends[watcher.name] << backend_name
|
770
|
+
else
|
771
|
+
log.info "synapse: restart required because we have a new backend #{watcher.name}/#{backend_name}"
|
714
772
|
@restart_required = true
|
715
|
-
return
|
716
773
|
end
|
717
|
-
|
718
|
-
enabled_backends[watcher.name] << backend_name
|
719
774
|
end
|
720
775
|
end
|
721
776
|
|
722
777
|
# actually enable the enabled backends, and disable the disabled ones
|
723
778
|
cur_backends.each do |section, backends|
|
724
779
|
backends.each do |backend|
|
725
|
-
if enabled_backends[
|
780
|
+
if enabled_backends.fetch(section, []).include? backend
|
726
781
|
command = "enable server #{section}/#{backend}\n"
|
727
782
|
else
|
728
783
|
command = "disable server #{section}/#{backend}\n"
|
@@ -736,12 +791,10 @@ module Synapse
|
|
736
791
|
rescue StandardError => e
|
737
792
|
log.warn "synapse: unknown error writing to socket"
|
738
793
|
@restart_required = true
|
739
|
-
return
|
740
794
|
else
|
741
795
|
unless output == "\n"
|
742
796
|
log.warn "synapse: socket command #{command} failed: #{output}"
|
743
797
|
@restart_required = true
|
744
|
-
return
|
745
798
|
end
|
746
799
|
end
|
747
800
|
end
|
@@ -767,18 +820,24 @@ module Synapse
|
|
767
820
|
end
|
768
821
|
end
|
769
822
|
|
770
|
-
# restarts haproxy
|
823
|
+
# restarts haproxy if the time is right
|
771
824
|
def restart
|
772
|
-
|
773
|
-
|
774
|
-
|
825
|
+
if @time < @next_restart
|
826
|
+
log.info "synapse: at time #{@time} waiting until #{@next_restart} to restart"
|
827
|
+
return
|
828
|
+
end
|
829
|
+
|
830
|
+
@next_restart = @time + @restart_interval
|
831
|
+
@next_restart += rand(@restart_jitter * @restart_interval + 1)
|
775
832
|
|
776
833
|
# do the actual restart
|
777
834
|
res = `#{opts['reload_command']}`.chomp
|
778
|
-
|
835
|
+
unless $?.success?
|
836
|
+
log.error "failed to reload haproxy via #{opts['reload_command']}: #{res}"
|
837
|
+
return
|
838
|
+
end
|
779
839
|
log.info "synapse: restarted haproxy"
|
780
840
|
|
781
|
-
@last_restart = Time.now()
|
782
841
|
@restart_required = false
|
783
842
|
end
|
784
843
|
|
@@ -791,5 +850,43 @@ module Synapse
|
|
791
850
|
|
792
851
|
return name
|
793
852
|
end
|
853
|
+
|
854
|
+
def update_state_file(watchers)
|
855
|
+
log.info "synapse: writing state file"
|
856
|
+
|
857
|
+
timestamp = Time.now.to_i
|
858
|
+
|
859
|
+
# Remove stale backends
|
860
|
+
@seen.each do |watcher_name, backends|
|
861
|
+
backends.each do |backend_name, backend|
|
862
|
+
ts = backend.fetch('timestamp', 0)
|
863
|
+
delta = (timestamp - ts).abs
|
864
|
+
if delta > @state_file_ttl
|
865
|
+
log.info "synapse: expiring #{backend_name} with age #{delta}"
|
866
|
+
backends.delete(backend_name)
|
867
|
+
end
|
868
|
+
end
|
869
|
+
end
|
870
|
+
|
871
|
+
# Remove any services which no longer have any backends
|
872
|
+
@seen = @seen.reject{|watcher_name, backends| backends.keys.length == 0}
|
873
|
+
|
874
|
+
# Add backends from watchers
|
875
|
+
watchers.each do |watcher|
|
876
|
+
unless @seen.key?(watcher.name)
|
877
|
+
@seen[watcher.name] = {}
|
878
|
+
end
|
879
|
+
|
880
|
+
watcher.backends.each do |backend|
|
881
|
+
backend_name = construct_name(backend)
|
882
|
+
@seen[watcher.name][backend_name] = backend.merge('timestamp' => timestamp)
|
883
|
+
end
|
884
|
+
end
|
885
|
+
|
886
|
+
# Atomically write new state file
|
887
|
+
tmp_state_file_path = @state_file_path + ".tmp"
|
888
|
+
File.write(tmp_state_file_path, JSON.pretty_generate(@seen))
|
889
|
+
FileUtils.mv(tmp_state_file_path, @state_file_path)
|
890
|
+
end
|
794
891
|
end
|
795
892
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'set'
|
1
2
|
require 'synapse/log'
|
2
3
|
|
3
4
|
module Synapse
|
@@ -42,6 +43,10 @@ module Synapse
|
|
42
43
|
|
43
44
|
@keep_default_servers = opts['keep_default_servers'] || false
|
44
45
|
|
46
|
+
# If there are no default servers and a watcher reports no backends, then
|
47
|
+
# use the previous backends that we already know about.
|
48
|
+
@use_previous_backends = opts.fetch('use_previous_backends', true)
|
49
|
+
|
45
50
|
# set a flag used to tell the watchers to exit
|
46
51
|
# this is not used in every watcher
|
47
52
|
@should_exit = false
|
@@ -95,13 +100,46 @@ module Synapse
|
|
95
100
|
end
|
96
101
|
|
97
102
|
def set_backends(new_backends)
|
98
|
-
|
99
|
-
|
103
|
+
# Aggregate and deduplicate all potential backend service instances.
|
104
|
+
new_backends = (new_backends + @default_servers) if @keep_default_servers
|
105
|
+
new_backends = new_backends.uniq {|b|
|
106
|
+
[b['host'], b['port'], b.fetch('name', '')]
|
107
|
+
}
|
108
|
+
|
109
|
+
if new_backends.to_set == @backends.to_set
|
110
|
+
return false
|
111
|
+
end
|
112
|
+
|
113
|
+
if new_backends.empty?
|
114
|
+
if @default_servers.empty?
|
115
|
+
if @use_previous_backends
|
116
|
+
# Discard this update
|
117
|
+
log.warn "synapse: no backends for service #{@name} and no default" \
|
118
|
+
" servers for service #{@name}; using previous backends: #{@backends.inspect}"
|
119
|
+
return false
|
120
|
+
else
|
121
|
+
log.warn "synapse: no backends for service #{@name}, no default" \
|
122
|
+
" servers for service #{@name} and 'use_previous_backends' is disabled;" \
|
123
|
+
" dropping all backends"
|
124
|
+
@backends.clear
|
125
|
+
end
|
126
|
+
else
|
127
|
+
log.warn "synapse: no backends for service #{@name};" \
|
128
|
+
" using default servers: #{@default_servers.inspect}"
|
129
|
+
@backends = @default_servers
|
130
|
+
end
|
100
131
|
else
|
132
|
+
log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
|
101
133
|
@backends = new_backends
|
102
134
|
end
|
135
|
+
|
136
|
+
reconfigure!
|
137
|
+
|
138
|
+
return true
|
103
139
|
end
|
104
140
|
|
141
|
+
# Subclasses should not invoke this directly; it's only exposed so that it
|
142
|
+
# can be overridden in subclasses.
|
105
143
|
def reconfigure!
|
106
144
|
@synapse.reconfigure!
|
107
145
|
end
|
@@ -89,21 +89,7 @@ module Synapse
|
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
92
|
-
|
93
|
-
if @default_servers.empty?
|
94
|
-
log.warn "synapse: no backends and no default servers for service #{@name};" \
|
95
|
-
" using previous backends: #{@backends.inspect}"
|
96
|
-
else
|
97
|
-
log.warn "synapse: no backends for service #{@name};" \
|
98
|
-
" using default servers: #{@default_servers.inspect}"
|
99
|
-
@backends = @default_servers
|
100
|
-
end
|
101
|
-
else
|
102
|
-
log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
|
103
|
-
set_backends(new_backends)
|
104
|
-
end
|
105
|
-
|
106
|
-
reconfigure!
|
92
|
+
set_backends(new_backends)
|
107
93
|
end
|
108
94
|
end
|
109
95
|
end
|
@@ -23,16 +23,10 @@ module Synapse
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def watch
|
26
|
-
last_containers = []
|
27
26
|
until @should_exit
|
28
27
|
begin
|
29
28
|
start = Time.now
|
30
|
-
|
31
|
-
unless last_containers == current_containers
|
32
|
-
last_containers = current_containers
|
33
|
-
configure_backends(last_containers)
|
34
|
-
end
|
35
|
-
|
29
|
+
set_backends(containers)
|
36
30
|
sleep_until_next_check(start)
|
37
31
|
rescue Exception => e
|
38
32
|
log.warn "synapse: error in watcher thread: #{e.inspect}"
|
@@ -98,23 +92,5 @@ module Synapse
|
|
98
92
|
log.warn "synapse: error while polling for containers: #{e.inspect}"
|
99
93
|
[]
|
100
94
|
end
|
101
|
-
|
102
|
-
def configure_backends(new_backends)
|
103
|
-
if new_backends.empty?
|
104
|
-
if @default_servers.empty?
|
105
|
-
log.warn "synapse: no backends and no default servers for service #{@name};" \
|
106
|
-
" using previous backends: #{@backends.inspect}"
|
107
|
-
else
|
108
|
-
log.warn "synapse: no backends for service #{@name};" \
|
109
|
-
" using default servers: #{@default_servers.inspect}"
|
110
|
-
@backends = @default_servers
|
111
|
-
end
|
112
|
-
else
|
113
|
-
log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
|
114
|
-
set_backends(new_backends)
|
115
|
-
end
|
116
|
-
reconfigure!
|
117
|
-
end
|
118
|
-
|
119
95
|
end
|
120
96
|
end
|
@@ -1,25 +1,107 @@
|
|
1
|
-
require
|
1
|
+
require 'synapse/service_watcher/base'
|
2
|
+
require 'aws-sdk'
|
2
3
|
|
3
4
|
module Synapse
|
4
5
|
class EC2Watcher < BaseWatcher
|
6
|
+
|
7
|
+
attr_reader :check_interval
|
8
|
+
|
5
9
|
def start
|
6
|
-
|
7
|
-
|
8
|
-
|
10
|
+
region = @discovery['aws_region'] || ENV['AWS_REGION']
|
11
|
+
log.info "Connecting to EC2 region: #{region}"
|
12
|
+
|
13
|
+
@ec2 = AWS::EC2.new(
|
14
|
+
region: region,
|
15
|
+
access_key_id: @discovery['aws_access_key_id'] || ENV['AWS_ACCESS_KEY_ID'],
|
16
|
+
secret_access_key: @discovery['aws_secret_access_key'] || ENV['AWS_SECRET_ACCESS_KEY'] )
|
17
|
+
|
18
|
+
@check_interval = @discovery['check_interval'] || 15.0
|
19
|
+
|
20
|
+
log.info "synapse: ec2tag watcher looking for instances " +
|
21
|
+
"tagged with #{@discovery['tag_name']}=#{@discovery['tag_value']}"
|
22
|
+
|
23
|
+
@watcher = Thread.new { watch }
|
9
24
|
end
|
10
25
|
|
11
26
|
private
|
27
|
+
|
12
28
|
def validate_discovery_opts
|
29
|
+
# Required, via options only.
|
13
30
|
raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
|
14
|
-
unless @discovery['method'] == 'ec2tag'
|
15
|
-
raise ArgumentError, "a `server_port_override` option is required for ec2tag watchers" \
|
16
|
-
unless @server_port_override
|
17
|
-
raise ArgumentError, "missing aws credentials for service #{@name}" \
|
18
|
-
unless (@discovery['aws_key'] && @discovery['aws_secret'])
|
31
|
+
unless @discovery['method'] == 'ec2tag'
|
19
32
|
raise ArgumentError, "aws tag name is required for service #{@name}" \
|
20
33
|
unless @discovery['tag_name']
|
21
34
|
raise ArgumentError, "aws tag value required for service #{@name}" \
|
22
35
|
unless @discovery['tag_value']
|
36
|
+
|
37
|
+
# As we're only looking up instances with hostnames/IPs, need to
|
38
|
+
# be explicitly told which port the service we're balancing for listens on.
|
39
|
+
unless @haproxy['server_port_override']
|
40
|
+
raise ArgumentError,
|
41
|
+
"Missing server_port_override for service #{@name} - which port are backends listening on?"
|
42
|
+
end
|
43
|
+
|
44
|
+
unless @haproxy['server_port_override'].match(/^\d+$/)
|
45
|
+
raise ArgumentError, "Invalid server_port_override value"
|
46
|
+
end
|
47
|
+
|
48
|
+
# Required, but can use well-known environment variables.
|
49
|
+
%w[aws_access_key_id aws_secret_access_key aws_region].each do |attr|
|
50
|
+
unless (@discovery[attr] || ENV[attr.upcase])
|
51
|
+
raise ArgumentError, "Missing #{attr} option or #{attr.upcase} environment variable"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def watch
|
57
|
+
until @should_exit
|
58
|
+
begin
|
59
|
+
start = Time.now
|
60
|
+
if set_backends(discover_instances)
|
61
|
+
log.info "synapse: ec2tag watcher backends have changed."
|
62
|
+
end
|
63
|
+
sleep_until_next_check(start)
|
64
|
+
rescue Exception => e
|
65
|
+
log.warn "synapse: error in ec2tag watcher thread: #{e.inspect}"
|
66
|
+
log.warn e.backtrace
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
log.info "synapse: ec2tag watcher exited successfully"
|
71
|
+
end
|
72
|
+
|
73
|
+
def sleep_until_next_check(start_time)
|
74
|
+
sleep_time = check_interval - (Time.now - start_time)
|
75
|
+
if sleep_time > 0.0
|
76
|
+
sleep(sleep_time)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def discover_instances
|
81
|
+
AWS.memoize do
|
82
|
+
instances = instances_with_tags(@discovery['tag_name'], @discovery['tag_value'])
|
83
|
+
|
84
|
+
new_backends = []
|
85
|
+
|
86
|
+
# choice of private_dns_name, dns_name, private_ip_address or
|
87
|
+
# ip_address, for now, just stick with the private fields.
|
88
|
+
instances.each do |instance|
|
89
|
+
new_backends << {
|
90
|
+
'name' => instance.private_dns_name,
|
91
|
+
'host' => instance.private_ip_address,
|
92
|
+
'port' => @haproxy['server_port_override'],
|
93
|
+
}
|
94
|
+
end
|
95
|
+
|
96
|
+
new_backends
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def instances_with_tags(tag_name, tag_value)
|
101
|
+
@ec2.instances
|
102
|
+
.tagged(tag_name)
|
103
|
+
.tagged_values(tag_value)
|
104
|
+
.select { |i| i.status == :running }
|
23
105
|
end
|
24
106
|
end
|
25
107
|
end
|