synapse 0.11.1 → 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,12 @@
1
+ require 'fileutils'
2
+ require 'json'
1
3
  require 'synapse/log'
2
4
  require 'socket'
3
5
 
4
6
  module Synapse
5
7
  class Haproxy
6
8
  include Logging
7
- attr_reader :opts
9
+ attr_reader :opts, :name
8
10
 
9
11
  # these come from the documentation for haproxy 1.5
10
12
  # http://haproxy.1wt.eu/download/1.5/doc/configuration.txt
@@ -43,6 +45,7 @@ module Synapse
43
45
  "option abortonclose",
44
46
  "option accept-invalid-http-response",
45
47
  "option allbackups",
48
+ "option allredisp",
46
49
  "option checkcache",
47
50
  "option forceclose",
48
51
  "option forwardfor",
@@ -173,6 +176,7 @@ module Synapse
173
176
  "option accept-invalid-http-request",
174
177
  "option accept-invalid-http-response",
175
178
  "option allbackups",
179
+ "option allredisp",
176
180
  "option checkcache",
177
181
  "option clitcpka",
178
182
  "option contstats",
@@ -386,6 +390,7 @@ module Synapse
386
390
  "option accept-invalid-http-request",
387
391
  "option accept-invalid-http-response",
388
392
  "option allbackups",
393
+ "option allredisp",
389
394
  "option checkcache",
390
395
  "option clitcpka",
391
396
  "option contstats",
@@ -523,20 +528,53 @@ module Synapse
523
528
  end
524
529
 
525
530
  @opts = opts
531
+ @name = 'haproxy'
532
+
533
+ @opts['do_writes'] = true unless @opts.key?('do_writes')
534
+ @opts['do_socket'] = true unless @opts.key?('do_socket')
535
+ @opts['do_reloads'] = true unless @opts.key?('do_reloads')
526
536
 
527
537
  # how to restart haproxy
528
- @restart_interval = 2
538
+ @restart_interval = @opts.fetch('restart_interval', 2).to_i
539
+ @restart_jitter = @opts.fetch('restart_jitter', 0).to_f
529
540
  @restart_required = true
530
- @last_restart = Time.new(0)
541
+
542
+ # virtual clock bookkeeping for controlling how often haproxy restarts
543
+ @time = 0
544
+ @next_restart = @time
531
545
 
532
546
  # a place to store the parsed haproxy config from each watcher
533
547
  @watcher_configs = {}
548
+
549
+ @state_file_path = @opts['state_file_path']
550
+ @state_file_ttl = @opts.fetch('state_file_ttl', 60 * 60 * 24).to_i
551
+ @seen = {}
552
+
553
+ unless @state_file_path.nil?
554
+ begin
555
+ @seen = JSON.load(File.read(@state_file_path))
556
+ rescue StandardError => e
557
+ # It's ok if the state file doesn't exist
558
+ end
559
+ end
560
+ end
561
+
562
+ def tick(watchers)
563
+ if @time % 60 == 0 && !@state_file_path.nil?
564
+ update_state_file(watchers)
565
+ end
566
+
567
+ @time += 1
568
+
569
+ # We potentially have to restart if the restart was rate limited
570
+ # in the original call to update_config
571
+ restart if @opts['do_reloads'] && @restart_required
534
572
  end
535
573
 
536
574
  def update_config(watchers)
537
575
  # if we support updating backends, try that whenever possible
538
576
  if @opts['do_socket']
539
- update_backends(watchers) unless @restart_required
577
+ update_backends(watchers)
540
578
  else
541
579
  @restart_required = true
542
580
  end
@@ -655,17 +693,35 @@ module Synapse
655
693
  end
656
694
 
657
695
  def generate_backend_stanza(watcher, config)
696
+ backends = {}
697
+
698
+ # The ordering here is important. First we add all the backends in the
699
+ # disabled state...
700
+ @seen.fetch(watcher.name, []).each do |backend_name, backend|
701
+ backends[backend_name] = backend.merge('enabled' => false)
702
+ end
703
+
704
+ # ... and then we overwite any backends that the watchers know about,
705
+ # setting the enabled state.
706
+ watcher.backends.each do |backend|
707
+ backend_name = construct_name(backend)
708
+ backends[backend_name] = backend.merge('enabled' => true)
709
+ end
710
+
658
711
  if watcher.backends.empty?
659
- log.warn "synapse: no backends found for watcher #{watcher.name}"
712
+ log.debug "synapse: no backends found for watcher #{watcher.name}"
660
713
  end
661
714
 
662
715
  stanza = [
663
716
  "\nbackend #{watcher.name}",
664
717
  config.map {|c| "\t#{c}"},
665
- watcher.backends.shuffle.map {|backend|
666
- backend_name = construct_name(backend)
667
- "\tserver #{backend_name} #{backend['host']}:#{backend['port']} " \
668
- "cookie #{backend_name} #{watcher.haproxy['server_options']}" }
718
+ backends.keys.shuffle.map {|backend_name|
719
+ backend = backends[backend_name]
720
+ b = "\tserver #{backend_name} #{backend['host']}:#{backend['port']}"
721
+ b = "#{b} cookie #{backend_name}" unless config.include?('mode tcp')
722
+ b = "#{b} #{watcher.haproxy['server_options']}"
723
+ b = "#{b} disabled" unless backend['enabled']
724
+ b }
669
725
  ]
670
726
  end
671
727
 
@@ -702,27 +758,26 @@ module Synapse
702
758
  next if watcher.backends.empty?
703
759
 
704
760
  unless cur_backends.include? watcher.name
705
- log.debug "synapse: restart required because we added new section #{watcher.name}"
761
+ log.info "synapse: restart required because we added new section #{watcher.name}"
706
762
  @restart_required = true
707
- return
763
+ next
708
764
  end
709
765
 
710
766
  watcher.backends.each do |backend|
711
767
  backend_name = construct_name(backend)
712
- unless cur_backends[watcher.name].include? backend_name
713
- log.debug "synapse: restart required because we have a new backend #{watcher.name}/#{backend_name}"
768
+ if cur_backends[watcher.name].include? backend_name
769
+ enabled_backends[watcher.name] << backend_name
770
+ else
771
+ log.info "synapse: restart required because we have a new backend #{watcher.name}/#{backend_name}"
714
772
  @restart_required = true
715
- return
716
773
  end
717
-
718
- enabled_backends[watcher.name] << backend_name
719
774
  end
720
775
  end
721
776
 
722
777
  # actually enable the enabled backends, and disable the disabled ones
723
778
  cur_backends.each do |section, backends|
724
779
  backends.each do |backend|
725
- if enabled_backends[section].include? backend
780
+ if enabled_backends.fetch(section, []).include? backend
726
781
  command = "enable server #{section}/#{backend}\n"
727
782
  else
728
783
  command = "disable server #{section}/#{backend}\n"
@@ -736,12 +791,10 @@ module Synapse
736
791
  rescue StandardError => e
737
792
  log.warn "synapse: unknown error writing to socket"
738
793
  @restart_required = true
739
- return
740
794
  else
741
795
  unless output == "\n"
742
796
  log.warn "synapse: socket command #{command} failed: #{output}"
743
797
  @restart_required = true
744
- return
745
798
  end
746
799
  end
747
800
  end
@@ -767,18 +820,24 @@ module Synapse
767
820
  end
768
821
  end
769
822
 
770
- # restarts haproxy
823
+ # restarts haproxy if the time is right
771
824
  def restart
772
- # sleep if we restarted too recently
773
- delay = (@last_restart - Time.now) + @restart_interval
774
- sleep(delay) if delay > 0
825
+ if @time < @next_restart
826
+ log.info "synapse: at time #{@time} waiting until #{@next_restart} to restart"
827
+ return
828
+ end
829
+
830
+ @next_restart = @time + @restart_interval
831
+ @next_restart += rand(@restart_jitter * @restart_interval + 1)
775
832
 
776
833
  # do the actual restart
777
834
  res = `#{opts['reload_command']}`.chomp
778
- raise "failed to reload haproxy via #{opts['reload_command']}: #{res}" unless $?.success?
835
+ unless $?.success?
836
+ log.error "failed to reload haproxy via #{opts['reload_command']}: #{res}"
837
+ return
838
+ end
779
839
  log.info "synapse: restarted haproxy"
780
840
 
781
- @last_restart = Time.now()
782
841
  @restart_required = false
783
842
  end
784
843
 
@@ -791,5 +850,43 @@ module Synapse
791
850
 
792
851
  return name
793
852
  end
853
+
854
+ def update_state_file(watchers)
855
+ log.info "synapse: writing state file"
856
+
857
+ timestamp = Time.now.to_i
858
+
859
+ # Remove stale backends
860
+ @seen.each do |watcher_name, backends|
861
+ backends.each do |backend_name, backend|
862
+ ts = backend.fetch('timestamp', 0)
863
+ delta = (timestamp - ts).abs
864
+ if delta > @state_file_ttl
865
+ log.info "synapse: expiring #{backend_name} with age #{delta}"
866
+ backends.delete(backend_name)
867
+ end
868
+ end
869
+ end
870
+
871
+ # Remove any services which no longer have any backends
872
+ @seen = @seen.reject{|watcher_name, backends| backends.keys.length == 0}
873
+
874
+ # Add backends from watchers
875
+ watchers.each do |watcher|
876
+ unless @seen.key?(watcher.name)
877
+ @seen[watcher.name] = {}
878
+ end
879
+
880
+ watcher.backends.each do |backend|
881
+ backend_name = construct_name(backend)
882
+ @seen[watcher.name][backend_name] = backend.merge('timestamp' => timestamp)
883
+ end
884
+ end
885
+
886
+ # Atomically write new state file
887
+ tmp_state_file_path = @state_file_path + ".tmp"
888
+ File.write(tmp_state_file_path, JSON.pretty_generate(@seen))
889
+ FileUtils.mv(tmp_state_file_path, @state_file_path)
890
+ end
794
891
  end
795
892
  end
@@ -1,3 +1,4 @@
1
+ require 'set'
1
2
  require 'synapse/log'
2
3
 
3
4
  module Synapse
@@ -42,6 +43,10 @@ module Synapse
42
43
 
43
44
  @keep_default_servers = opts['keep_default_servers'] || false
44
45
 
46
+ # If there are no default servers and a watcher reports no backends, then
47
+ # use the previous backends that we already know about.
48
+ @use_previous_backends = opts.fetch('use_previous_backends', true)
49
+
45
50
  # set a flag used to tell the watchers to exit
46
51
  # this is not used in every watcher
47
52
  @should_exit = false
@@ -95,13 +100,46 @@ module Synapse
95
100
  end
96
101
 
97
102
  def set_backends(new_backends)
98
- if @keep_default_servers
99
- @backends = @default_servers + new_backends
103
+ # Aggregate and deduplicate all potential backend service instances.
104
+ new_backends = (new_backends + @default_servers) if @keep_default_servers
105
+ new_backends = new_backends.uniq {|b|
106
+ [b['host'], b['port'], b.fetch('name', '')]
107
+ }
108
+
109
+ if new_backends.to_set == @backends.to_set
110
+ return false
111
+ end
112
+
113
+ if new_backends.empty?
114
+ if @default_servers.empty?
115
+ if @use_previous_backends
116
+ # Discard this update
117
+ log.warn "synapse: no backends for service #{@name} and no default" \
118
+ " servers for service #{@name}; using previous backends: #{@backends.inspect}"
119
+ return false
120
+ else
121
+ log.warn "synapse: no backends for service #{@name}, no default" \
122
+ " servers for service #{@name} and 'use_previous_backends' is disabled;" \
123
+ " dropping all backends"
124
+ @backends.clear
125
+ end
126
+ else
127
+ log.warn "synapse: no backends for service #{@name};" \
128
+ " using default servers: #{@default_servers.inspect}"
129
+ @backends = @default_servers
130
+ end
100
131
  else
132
+ log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
101
133
  @backends = new_backends
102
134
  end
135
+
136
+ reconfigure!
137
+
138
+ return true
103
139
  end
104
140
 
141
+ # Subclasses should not invoke this directly; it's only exposed so that it
142
+ # can be overridden in subclasses.
105
143
  def reconfigure!
106
144
  @synapse.reconfigure!
107
145
  end
@@ -89,21 +89,7 @@ module Synapse
89
89
  end
90
90
  end
91
91
 
92
- if new_backends.empty?
93
- if @default_servers.empty?
94
- log.warn "synapse: no backends and no default servers for service #{@name};" \
95
- " using previous backends: #{@backends.inspect}"
96
- else
97
- log.warn "synapse: no backends for service #{@name};" \
98
- " using default servers: #{@default_servers.inspect}"
99
- @backends = @default_servers
100
- end
101
- else
102
- log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
103
- set_backends(new_backends)
104
- end
105
-
106
- reconfigure!
92
+ set_backends(new_backends)
107
93
  end
108
94
  end
109
95
  end
@@ -23,16 +23,10 @@ module Synapse
23
23
  end
24
24
 
25
25
  def watch
26
- last_containers = []
27
26
  until @should_exit
28
27
  begin
29
28
  start = Time.now
30
- current_containers = containers
31
- unless last_containers == current_containers
32
- last_containers = current_containers
33
- configure_backends(last_containers)
34
- end
35
-
29
+ set_backends(containers)
36
30
  sleep_until_next_check(start)
37
31
  rescue Exception => e
38
32
  log.warn "synapse: error in watcher thread: #{e.inspect}"
@@ -98,23 +92,5 @@ module Synapse
98
92
  log.warn "synapse: error while polling for containers: #{e.inspect}"
99
93
  []
100
94
  end
101
-
102
- def configure_backends(new_backends)
103
- if new_backends.empty?
104
- if @default_servers.empty?
105
- log.warn "synapse: no backends and no default servers for service #{@name};" \
106
- " using previous backends: #{@backends.inspect}"
107
- else
108
- log.warn "synapse: no backends for service #{@name};" \
109
- " using default servers: #{@default_servers.inspect}"
110
- @backends = @default_servers
111
- end
112
- else
113
- log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
114
- set_backends(new_backends)
115
- end
116
- reconfigure!
117
- end
118
-
119
95
  end
120
96
  end
@@ -1,25 +1,107 @@
1
- require "synapse/service_watcher/base"
1
+ require 'synapse/service_watcher/base'
2
+ require 'aws-sdk'
2
3
 
3
4
  module Synapse
4
5
  class EC2Watcher < BaseWatcher
6
+
7
+ attr_reader :check_interval
8
+
5
9
  def start
6
- # connect to ec2
7
- # find all servers whose @discovery['tag_name'] matches @discovery['tag_value']
8
- # call @synapse.configure
10
+ region = @discovery['aws_region'] || ENV['AWS_REGION']
11
+ log.info "Connecting to EC2 region: #{region}"
12
+
13
+ @ec2 = AWS::EC2.new(
14
+ region: region,
15
+ access_key_id: @discovery['aws_access_key_id'] || ENV['AWS_ACCESS_KEY_ID'],
16
+ secret_access_key: @discovery['aws_secret_access_key'] || ENV['AWS_SECRET_ACCESS_KEY'] )
17
+
18
+ @check_interval = @discovery['check_interval'] || 15.0
19
+
20
+ log.info "synapse: ec2tag watcher looking for instances " +
21
+ "tagged with #{@discovery['tag_name']}=#{@discovery['tag_value']}"
22
+
23
+ @watcher = Thread.new { watch }
9
24
  end
10
25
 
11
26
  private
27
+
12
28
  def validate_discovery_opts
29
+ # Required, via options only.
13
30
  raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
14
- unless @discovery['method'] == 'ec2tag'
15
- raise ArgumentError, "a `server_port_override` option is required for ec2tag watchers" \
16
- unless @server_port_override
17
- raise ArgumentError, "missing aws credentials for service #{@name}" \
18
- unless (@discovery['aws_key'] && @discovery['aws_secret'])
31
+ unless @discovery['method'] == 'ec2tag'
19
32
  raise ArgumentError, "aws tag name is required for service #{@name}" \
20
33
  unless @discovery['tag_name']
21
34
  raise ArgumentError, "aws tag value required for service #{@name}" \
22
35
  unless @discovery['tag_value']
36
+
37
+ # As we're only looking up instances with hostnames/IPs, need to
38
+ # be explicitly told which port the service we're balancing for listens on.
39
+ unless @haproxy['server_port_override']
40
+ raise ArgumentError,
41
+ "Missing server_port_override for service #{@name} - which port are backends listening on?"
42
+ end
43
+
44
+ unless @haproxy['server_port_override'].match(/^\d+$/)
45
+ raise ArgumentError, "Invalid server_port_override value"
46
+ end
47
+
48
+ # Required, but can use well-known environment variables.
49
+ %w[aws_access_key_id aws_secret_access_key aws_region].each do |attr|
50
+ unless (@discovery[attr] || ENV[attr.upcase])
51
+ raise ArgumentError, "Missing #{attr} option or #{attr.upcase} environment variable"
52
+ end
53
+ end
54
+ end
55
+
56
+ def watch
57
+ until @should_exit
58
+ begin
59
+ start = Time.now
60
+ if set_backends(discover_instances)
61
+ log.info "synapse: ec2tag watcher backends have changed."
62
+ end
63
+ sleep_until_next_check(start)
64
+ rescue Exception => e
65
+ log.warn "synapse: error in ec2tag watcher thread: #{e.inspect}"
66
+ log.warn e.backtrace
67
+ end
68
+ end
69
+
70
+ log.info "synapse: ec2tag watcher exited successfully"
71
+ end
72
+
73
+ def sleep_until_next_check(start_time)
74
+ sleep_time = check_interval - (Time.now - start_time)
75
+ if sleep_time > 0.0
76
+ sleep(sleep_time)
77
+ end
78
+ end
79
+
80
+ def discover_instances
81
+ AWS.memoize do
82
+ instances = instances_with_tags(@discovery['tag_name'], @discovery['tag_value'])
83
+
84
+ new_backends = []
85
+
86
+ # choice of private_dns_name, dns_name, private_ip_address or
87
+ # ip_address, for now, just stick with the private fields.
88
+ instances.each do |instance|
89
+ new_backends << {
90
+ 'name' => instance.private_dns_name,
91
+ 'host' => instance.private_ip_address,
92
+ 'port' => @haproxy['server_port_override'],
93
+ }
94
+ end
95
+
96
+ new_backends
97
+ end
98
+ end
99
+
100
+ def instances_with_tags(tag_name, tag_value)
101
+ @ec2.instances
102
+ .tagged(tag_name)
103
+ .tagged_values(tag_value)
104
+ .select { |i| i.status == :running }
23
105
  end
24
106
  end
25
107
  end