democracyworks-synapse 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,24 @@
1
+ module Synapse
2
+ module Logging
3
+
4
+ def log
5
+ @logger ||= Logging.logger_for(self.class.name)
6
+ end
7
+
8
+ # Use a hash class-ivar to cache a unique Logger per class:
9
+ @loggers = {}
10
+
11
+ class << self
12
+ def logger_for(classname)
13
+ @loggers[classname] ||= configure_logger_for(classname)
14
+ end
15
+
16
+ def configure_logger_for(classname)
17
+ logger = Logger.new(STDERR)
18
+ logger.level = Logger::INFO unless ENV['DEBUG']
19
+ logger.progname = classname
20
+ return logger
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,32 @@
1
+ require "synapse/service_watcher/base"
2
+ require "synapse/service_watcher/zookeeper"
3
+ require "synapse/service_watcher/ec2tag"
4
+ require "synapse/service_watcher/dns"
5
+ require "synapse/service_watcher/docker"
6
+
7
+ module Synapse
8
+ class ServiceWatcher
9
+
10
+ @watchers = {
11
+ 'base'=>BaseWatcher,
12
+ 'zookeeper'=>ZookeeperWatcher,
13
+ 'ec2tag'=>EC2Watcher,
14
+ 'dns' => DnsWatcher,
15
+ 'docker' => DockerWatcher
16
+ }
17
+
18
+ # the method which actually dispatches watcher creation requests
19
+ def self.create(name, opts, synapse)
20
+ opts['name'] = name
21
+
22
+ raise ArgumentError, "Missing discovery method when trying to create watcher" \
23
+ unless opts.has_key?('discovery') && opts['discovery'].has_key?('method')
24
+
25
+ discovery_method = opts['discovery']['method']
26
+ raise ArgumentError, "Invalid discovery method #{discovery_method}" \
27
+ unless @watchers.has_key?(discovery_method)
28
+
29
+ return @watchers[discovery_method].new(opts, synapse)
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,105 @@
1
+ require 'synapse/log'
2
+
3
+ module Synapse
4
+ class BaseWatcher
5
+ include Logging
6
+
7
+ LEADER_WARN_INTERVAL = 30
8
+
9
+ attr_reader :name, :haproxy
10
+
11
+ def initialize(opts={}, synapse)
12
+ super()
13
+
14
+ @synapse = synapse
15
+
16
+ # set required service parameters
17
+ %w{name discovery haproxy}.each do |req|
18
+ raise ArgumentError, "missing required option #{req}" unless opts[req]
19
+ end
20
+
21
+ @name = opts['name']
22
+ @discovery = opts['discovery']
23
+
24
+ @leader_election = opts['leader_election'] || false
25
+ @leader_last_warn = Time.now - LEADER_WARN_INTERVAL
26
+
27
+ # the haproxy config
28
+ @haproxy = opts['haproxy']
29
+ @haproxy['server_options'] ||= ""
30
+ @haproxy['server_port_override'] ||= nil
31
+ %w{backend frontend listen}.each do |sec|
32
+ @haproxy[sec] ||= []
33
+ end
34
+
35
+ unless @haproxy.include?('port')
36
+ log.warn "synapse: service #{name}: haproxy config does not include a port; only backend sections for the service will be created; you must move traffic there manually using configuration in `extra_sections`"
37
+ end
38
+
39
+ # set initial backends to default servers, if any
40
+ @default_servers = opts['default_servers'] || []
41
+ @backends = @default_servers
42
+
43
+ @keep_default_servers = opts['keep_default_servers'] || false
44
+
45
+ # set a flag used to tell the watchers to exit
46
+ # this is not used in every watcher
47
+ @should_exit = false
48
+
49
+ validate_discovery_opts
50
+ end
51
+
52
+ # this should be overridden to actually start your watcher
53
+ def start
54
+ log.info "synapse: starting stub watcher; this means doing nothing at all!"
55
+ end
56
+
57
+ # this should be overridden to actually stop your watcher if necessary
58
+ # if you are running a thread, your loop should run `until @should_exit`
59
+ def stop
60
+ log.info "synapse: stopping watcher #{self.name} using default stop handler"
61
+ @should_exit = true
62
+ end
63
+
64
+ # this should be overridden to do a health check of the watcher
65
+ def ping?
66
+ true
67
+ end
68
+
69
+ def backends
70
+ if @leader_election
71
+ if @backends.all?{|b| b.key?('id') && b['id']}
72
+ smallest = @backends.sort_by{ |b| b['id']}.first
73
+ log.debug "synapse: leader election chose one of #{@backends.count} backends " \
74
+ "(#{smallest['host']}:#{smallest['port']} with id #{smallest['id']})"
75
+
76
+ return [smallest]
77
+ elsif (Time.now - @leader_last_warn) > LEADER_WARN_INTERVAL
78
+ log.warn "synapse: service #{@name}: leader election failed; not all backends include an id"
79
+ @leader_last_warn = Time.now
80
+ end
81
+
82
+ # if leader election fails, return no backends
83
+ return []
84
+ end
85
+
86
+ return @backends
87
+ end
88
+
89
+ private
90
+ def validate_discovery_opts
91
+ raise ArgumentError, "invalid discovery method '#{@discovery['method']}' for base watcher" \
92
+ unless @discovery['method'] == 'base'
93
+
94
+ log.warn "synapse: warning: a stub watcher with no default servers is pretty useless" if @default_servers.empty?
95
+ end
96
+
97
+ def set_backends(new_backends)
98
+ if @keep_default_servers
99
+ @backends = @default_servers + new_backends
100
+ else
101
+ @backends = new_backends
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,103 @@
1
+ require "synapse/service_watcher/base"
2
+
3
+ require 'thread'
4
+ require 'resolv'
5
+
6
+ module Synapse
7
+ class DnsWatcher < BaseWatcher
8
+ def start
9
+ @check_interval = @discovery['check_interval'] || 30.0
10
+ @nameserver = @discovery['nameserver']
11
+
12
+ @watcher = Thread.new do
13
+ watch
14
+ end
15
+ end
16
+
17
+ def ping?
18
+ !(resolver.getaddresses('airbnb.com').empty?)
19
+ end
20
+
21
+ private
22
+ def validate_discovery_opts
23
+ raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
24
+ unless @discovery['method'] == 'dns'
25
+ raise ArgumentError, "a non-empty list of servers is required" \
26
+ if @discovery['servers'].empty?
27
+ end
28
+
29
+ def watch
30
+ last_resolution = resolve_servers
31
+ configure_backends(last_resolution)
32
+ until @should_exit
33
+ begin
34
+ start = Time.now
35
+ current_resolution = resolve_servers
36
+ unless last_resolution == current_resolution
37
+ last_resolution = current_resolution
38
+ configure_backends(last_resolution)
39
+ end
40
+
41
+ sleep_until_next_check(start)
42
+ rescue => e
43
+ log.warn "Error in watcher thread: #{e.inspect}"
44
+ log.warn e.backtrace
45
+ end
46
+ end
47
+
48
+ log.info "synapse: dns watcher exited successfully"
49
+ end
50
+
51
+ def sleep_until_next_check(start_time)
52
+ sleep_time = @check_interval - (Time.now - start_time)
53
+ if sleep_time > 0.0
54
+ sleep(sleep_time)
55
+ end
56
+ end
57
+
58
+ def resolve_servers
59
+ resolver.tap do |dns|
60
+ resolution = @discovery['servers'].map do |server|
61
+ addresses = dns.getaddresses(server['host']).map(&:to_s)
62
+ [server, addresses.sort]
63
+ end
64
+
65
+ return resolution
66
+ end
67
+ rescue => e
68
+ log.warn "Error while resolving host names: #{e.inspect}"
69
+ []
70
+ end
71
+
72
+ def resolver
73
+ args = [{:nameserver => @nameserver}] if @nameserver
74
+ Resolv::DNS.open(*args)
75
+ end
76
+
77
+ def configure_backends(servers)
78
+ new_backends = servers.flat_map do |(server, addresses)|
79
+ addresses.map do |address|
80
+ {
81
+ 'host' => address,
82
+ 'port' => server['port']
83
+ }
84
+ end
85
+ end
86
+
87
+ if new_backends.empty?
88
+ if @default_servers.empty?
89
+ log.warn "synapse: no backends and no default servers for service #{@name};" \
90
+ " using previous backends: #{@backends.inspect}"
91
+ else
92
+ log.warn "synapse: no backends for service #{@name};" \
93
+ " using default servers: #{@default_servers.inspect}"
94
+ @backends = @default_servers
95
+ end
96
+ else
97
+ log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
98
+ set_backends(new_backends)
99
+ end
100
+ @synapse.reconfigure!
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,115 @@
1
+ require "synapse/service_watcher/base"
2
+ require 'docker'
3
+
4
+ module Synapse
5
+ class DockerWatcher < BaseWatcher
6
+ def start
7
+ @check_interval = @discovery['check_interval'] || 15.0
8
+ @watcher = Thread.new do
9
+ watch
10
+ end
11
+ end
12
+
13
+ private
14
+ def validate_discovery_opts
15
+ raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
16
+ unless @discovery['method'] == 'docker'
17
+ raise ArgumentError, "a non-empty list of servers is required" \
18
+ if @discovery['servers'].nil? or @discovery['servers'].empty?
19
+ raise ArgumentError, "non-empty image_name required" \
20
+ if @discovery['image_name'].nil? or @discovery['image_name'].empty?
21
+ raise ArgumentError, "container_port required" \
22
+ if @discovery['container_port'].nil?
23
+ end
24
+
25
+ def watch
26
+ last_containers = []
27
+ until @should_exit
28
+ begin
29
+ start = Time.now
30
+ current_containers = containers
31
+ unless last_containers == current_containers
32
+ last_containers = current_containers
33
+ configure_backends(last_containers)
34
+ end
35
+
36
+ sleep_until_next_check(start)
37
+ rescue Exception => e
38
+ log.warn "synapse: error in watcher thread: #{e.inspect}"
39
+ log.warn e.backtrace
40
+ end
41
+ end
42
+
43
+ log.info "synapse: docker watcher exited successfully"
44
+ end
45
+
46
+ def sleep_until_next_check(start_time)
47
+ sleep_time = @check_interval - (Time.now - start_time)
48
+ if sleep_time > 0.0
49
+ sleep(sleep_time)
50
+ end
51
+ end
52
+
53
+ def containers
54
+ backends = @discovery['servers'].map do |server|
55
+ Docker.url = "http://#{server['host']}:#{server['port'] || 4243}"
56
+ begin
57
+ cnts = Docker::Util.parse_json(Docker.connection.get('/containers/json', {}))
58
+ rescue => e
59
+ log.warn "synapse: error polling docker host #{Docker.url}: #{e.inspect}"
60
+ next []
61
+ end
62
+ cnts.each do |cnt|
63
+ pairs = nil
64
+ if cnt["Ports"].is_a?(String)
65
+ # "Ports" comes through (as of 0.6.5) as a string like "0.0.0.0:49153->6379/tcp, 0.0.0.0:49153->6379/tcp"
66
+ # Convert string to a map of container port to host port: {"7000"->"49158", "6379": "49159"}
67
+ pairs = cnt["Ports"].split(", ").collect do |v|
68
+ pair = v.split('->')
69
+ [ pair[1].rpartition("/").first, pair[0].rpartition(":").last ]
70
+ end
71
+ else
72
+ pairs = cnt["Ports"].collect do |v|
73
+ [v['PrivatePort'].to_s, v['PublicPort'].to_s]
74
+ end
75
+ end
76
+ cnt["Ports"] = Hash[pairs]
77
+ end
78
+ # Discover containers that match the image/port we're interested in
79
+ cnts = cnts.find_all do |cnt|
80
+ cnt["Image"].rpartition(":").first == @discovery["image_name"] \
81
+ and cnt["Ports"].has_key?(@discovery["container_port"].to_s())
82
+ end
83
+ cnts.map do |cnt|
84
+ {
85
+ 'name' => server['name'],
86
+ 'host' => server['host'],
87
+ 'port' => cnt["Ports"][@discovery["container_port"].to_s()]
88
+ }
89
+ end
90
+ end
91
+ backends.flatten
92
+ rescue => e
93
+ log.warn "synapse: error while polling for containers: #{e.inspect}"
94
+ []
95
+ end
96
+
97
+ def configure_backends(new_backends)
98
+ if new_backends.empty?
99
+ if @default_servers.empty?
100
+ log.warn "synapse: no backends and no default servers for service #{@name};" \
101
+ " using previous backends: #{@backends.inspect}"
102
+ else
103
+ log.warn "synapse: no backends for service #{@name};" \
104
+ " using default servers: #{@default_servers.inspect}"
105
+ @backends = @default_servers
106
+ end
107
+ else
108
+ log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
109
+ set_backends(new_backends)
110
+ end
111
+ @synapse.reconfigure!
112
+ end
113
+
114
+ end
115
+ end
@@ -0,0 +1,26 @@
1
+ require "synapse/service_watcher/base"
2
+
3
+ module Synapse
4
+ class EC2Watcher < BaseWatcher
5
+ def start
6
+ # connect to ec2
7
+ # find all servers whose @discovery['tag_name'] matches @discovery['tag_value']
8
+ # call @synapse.configure
9
+ end
10
+
11
+ private
12
+ def validate_discovery_opts
13
+ raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
14
+ unless @discovery['method'] == 'ec2tag'
15
+ raise ArgumentError, "a `server_port_override` option is required for ec2tag watchers" \
16
+ unless @server_port_override
17
+ raise ArgumentError, "missing aws credentials for service #{@name}" \
18
+ unless (@discovery['aws_key'] && @discovery['aws_secret'])
19
+ raise ArgumentError, "aws tag name is required for service #{@name}" \
20
+ unless @discovery['tag_name']
21
+ raise ArgumentError, "aws tag value required for service #{@name}" \
22
+ unless @discovery['tag_value']
23
+ end
24
+ end
25
+ end
26
+
@@ -0,0 +1,127 @@
1
+ require "synapse/service_watcher/base"
2
+
3
+ require 'zk'
4
+
5
+ module Synapse
6
+ class ZookeeperWatcher < BaseWatcher
7
+ NUMBERS_RE = /^\d+$/
8
+
9
+ def start
10
+ zk_hosts = @discovery['hosts'].shuffle.join(',')
11
+
12
+ log.info "synapse: starting ZK watcher #{@name} @ hosts: #{zk_hosts}, path: #{@discovery['path']}"
13
+ @should_exit = false
14
+ @zk = ZK.new(zk_hosts)
15
+
16
+ # call the callback to bootstrap the process
17
+ watcher_callback.call
18
+ end
19
+
20
+ def stop
21
+ log.warn "synapse: zookeeper watcher exiting"
22
+
23
+ @should_exit = true
24
+ @watcher.unsubscribe if defined? @watcher
25
+ @zk.close! if defined? @zk
26
+
27
+ log.info "synapse: zookeeper watcher cleaned up successfully"
28
+ end
29
+
30
+ def ping?
31
+ @zk.ping?
32
+ end
33
+
34
+ private
35
+ def validate_discovery_opts
36
+ raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
37
+ unless @discovery['method'] == 'zookeeper'
38
+ raise ArgumentError, "missing or invalid zookeeper host for service #{@name}" \
39
+ unless @discovery['hosts']
40
+ raise ArgumentError, "invalid zookeeper path for service #{@name}" \
41
+ unless @discovery['path']
42
+ end
43
+
44
+ # helper method that ensures that the discovery path exists
45
+ def create(path)
46
+ log.debug "synapse: creating ZK path: #{path}"
47
+ # recurse if the parent node does not exist
48
+ create File.dirname(path) unless @zk.exists? File.dirname(path)
49
+ @zk.create(path, ignore: :node_exists)
50
+ end
51
+
52
+ # find the current backends at the discovery path; sets @backends
53
+ def discover
54
+ log.info "synapse: discovering backends for service #{@name}"
55
+
56
+ new_backends = []
57
+ begin
58
+ @zk.children(@discovery['path'], :watch => true).each do |id|
59
+ node = @zk.get("#{@discovery['path']}/#{id}")
60
+
61
+ begin
62
+ host, port, name = deserialize_service_instance(node.first)
63
+ rescue StandardError => e
64
+ log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
65
+ else
66
+ server_port = @server_port_override ? @server_port_override : port
67
+
68
+ # find the numberic id in the node name; used for leader elections if enabled
69
+ numeric_id = id.split('_').last
70
+ numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
71
+
72
+ log.debug "synapse: discovered backend #{name} at #{host}:#{server_port} for service #{@name}"
73
+ new_backends << { 'name' => name, 'host' => host, 'port' => server_port, 'id' => numeric_id}
74
+ end
75
+ end
76
+ rescue ZK::Exceptions::NoNode
77
+ # the path must exist, otherwise watch callbacks will not work
78
+ create(@discovery['path'])
79
+ retry
80
+ end
81
+
82
+ if new_backends.empty?
83
+ if @default_servers.empty?
84
+ log.warn "synapse: no backends and no default servers for service #{@name}; using previous backends: #{@backends.inspect}"
85
+ else
86
+ log.warn "synapse: no backends for service #{@name}; using default servers: #{@default_servers.inspect}"
87
+ @backends = @default_servers
88
+ end
89
+ else
90
+ log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
91
+ set_backends(new_backends)
92
+ end
93
+ end
94
+
95
+ # sets up zookeeper callbacks if the data at the discovery path changes
96
+ def watch
97
+ return if @should_exit
98
+
99
+ @watcher.unsubscribe if defined? @watcher
100
+ @watcher = @zk.register(@discovery['path'], &watcher_callback)
101
+ end
102
+
103
+ # handles the event that a watched path has changed in zookeeper
104
+ def watcher_callback
105
+ @callback ||= Proc.new do |event|
106
+ # Set new watcher
107
+ watch
108
+ # Rediscover
109
+ discover
110
+ # send a message to calling class to reconfigure
111
+ @synapse.reconfigure!
112
+ end
113
+ end
114
+
115
+ # decode the data at a zookeeper endpoint
116
+ def deserialize_service_instance(data)
117
+ log.debug "synapse: deserializing process data"
118
+ decoded = JSON.parse(data)
119
+
120
+ host = decoded['host'] || (raise ValueError, 'instance json data does not have host key')
121
+ port = decoded['port'] || (raise ValueError, 'instance json data does not have port key')
122
+ name = decoded['name'] || nil
123
+
124
+ return host, port, name
125
+ end
126
+ end
127
+ end