synapse-aurora 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +23 -0
- data/.mailmap +3 -0
- data/.nix/Gemfile.nix +141 -0
- data/.nix/rubylibs.nix +42 -0
- data/.rspec +2 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/Makefile +6 -0
- data/README.md +339 -0
- data/Rakefile +8 -0
- data/bin/synapse +62 -0
- data/config/hostheader_test.json +71 -0
- data/config/svcdir_test.json +46 -0
- data/config/synapse.conf.json +90 -0
- data/config/synapse_services/service1.json +24 -0
- data/config/synapse_services/service2.json +24 -0
- data/default.nix +66 -0
- data/lib/synapse.rb +85 -0
- data/lib/synapse/base.rb +5 -0
- data/lib/synapse/haproxy.rb +797 -0
- data/lib/synapse/log.rb +24 -0
- data/lib/synapse/service_watcher.rb +36 -0
- data/lib/synapse/service_watcher/base.rb +109 -0
- data/lib/synapse/service_watcher/dns.rb +109 -0
- data/lib/synapse/service_watcher/docker.rb +120 -0
- data/lib/synapse/service_watcher/ec2tag.rb +133 -0
- data/lib/synapse/service_watcher/zookeeper.rb +153 -0
- data/lib/synapse/service_watcher/zookeeper_aurora.rb +76 -0
- data/lib/synapse/service_watcher/zookeeper_dns.rb +232 -0
- data/lib/synapse/version.rb +3 -0
- data/spec/lib/synapse/haproxy_spec.rb +32 -0
- data/spec/lib/synapse/service_watcher_base_spec.rb +55 -0
- data/spec/lib/synapse/service_watcher_docker_spec.rb +152 -0
- data/spec/lib/synapse/service_watcher_ec2tags_spec.rb +220 -0
- data/spec/spec_helper.rb +22 -0
- data/spec/support/configuration.rb +9 -0
- data/spec/support/minimum.conf.yaml +27 -0
- data/synapse.gemspec +33 -0
- metadata +227 -0
@@ -0,0 +1,153 @@
|
|
1
|
+
require "synapse/service_watcher/base"
|
2
|
+
|
3
|
+
require 'zk'
|
4
|
+
|
5
|
+
module Synapse
|
6
|
+
class ZookeeperWatcher < BaseWatcher
|
7
|
+
NUMBERS_RE = /^\d+$/
|
8
|
+
|
9
|
+
def start
|
10
|
+
@zk_hosts = @discovery['hosts'].shuffle.join(',')
|
11
|
+
|
12
|
+
@watcher = nil
|
13
|
+
@zk = nil
|
14
|
+
|
15
|
+
log.info "synapse: starting ZK watcher #{@name} @ hosts: #{@zk_hosts}, path: #{@discovery['path']}"
|
16
|
+
zk_connect
|
17
|
+
end
|
18
|
+
|
19
|
+
def stop
|
20
|
+
log.warn "synapse: zookeeper watcher exiting"
|
21
|
+
zk_cleanup
|
22
|
+
end
|
23
|
+
|
24
|
+
def ping?
|
25
|
+
@zk && @zk.connected?
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def validate_discovery_opts
|
31
|
+
raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
|
32
|
+
unless @discovery['method'] == 'zookeeper'
|
33
|
+
raise ArgumentError, "missing or invalid zookeeper host for service #{@name}" \
|
34
|
+
unless @discovery['hosts']
|
35
|
+
raise ArgumentError, "invalid zookeeper path for service #{@name}" \
|
36
|
+
unless @discovery['path']
|
37
|
+
end
|
38
|
+
|
39
|
+
# helper method that ensures that the discovery path exists
|
40
|
+
def create(path)
|
41
|
+
log.debug "synapse: creating ZK path: #{path}"
|
42
|
+
|
43
|
+
# recurse if the parent node does not exist
|
44
|
+
create File.dirname(path) unless @zk.exists? File.dirname(path)
|
45
|
+
@zk.create(path, ignore: :node_exists)
|
46
|
+
end
|
47
|
+
|
48
|
+
# find the current backends at the discovery path; sets @backends
|
49
|
+
def discover
|
50
|
+
log.info "synapse: discovering backends for service #{@name}"
|
51
|
+
|
52
|
+
new_backends = []
|
53
|
+
@zk.children(@discovery['path'], :watch => true).each do |id|
|
54
|
+
node = @zk.get("#{@discovery['path']}/#{id}")
|
55
|
+
|
56
|
+
begin
|
57
|
+
host, port, name = deserialize_service_instance(node.first)
|
58
|
+
rescue StandardError => e
|
59
|
+
log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
|
60
|
+
else
|
61
|
+
server_port = @server_port_override ? @server_port_override : port
|
62
|
+
|
63
|
+
# find the numberic id in the node name; used for leader elections if enabled
|
64
|
+
numeric_id = id.split('_').last
|
65
|
+
numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
|
66
|
+
|
67
|
+
log.debug "synapse: discovered backend #{name} at #{host}:#{server_port} for service #{@name}"
|
68
|
+
new_backends << { 'name' => name, 'host' => host, 'port' => server_port, 'id' => numeric_id}
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
if new_backends.empty?
|
73
|
+
if @default_servers.empty?
|
74
|
+
log.warn "synapse: no backends and no default servers for service #{@name}; using previous backends: #{@backends.inspect}"
|
75
|
+
else
|
76
|
+
log.warn "synapse: no backends for service #{@name}; using default servers: #{@default_servers.inspect}"
|
77
|
+
@backends = @default_servers
|
78
|
+
end
|
79
|
+
else
|
80
|
+
log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
|
81
|
+
set_backends(new_backends)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# sets up zookeeper callbacks if the data at the discovery path changes
|
86
|
+
def watch
|
87
|
+
return if @zk.nil?
|
88
|
+
|
89
|
+
@watcher.unsubscribe unless @watcher.nil?
|
90
|
+
@watcher = @zk.register(@discovery['path'], &watcher_callback)
|
91
|
+
|
92
|
+
# Verify that we actually set up the watcher.
|
93
|
+
unless @zk.exists?(@discovery['path'], :watch => true)
|
94
|
+
log.error "synapse: zookeeper watcher path #{@discovery['path']} does not exist!"
|
95
|
+
raise RuntimeError.new('could not set a ZK watch on a node that should exist')
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# handles the event that a watched path has changed in zookeeper
|
100
|
+
def watcher_callback
|
101
|
+
@callback ||= Proc.new do |event|
|
102
|
+
# Set new watcher
|
103
|
+
watch
|
104
|
+
# Rediscover
|
105
|
+
discover
|
106
|
+
# send a message to calling class to reconfigure
|
107
|
+
reconfigure!
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def zk_cleanup
|
112
|
+
log.info "synapse: zookeeper watcher cleaning up"
|
113
|
+
|
114
|
+
@watcher.unsubscribe unless @watcher.nil?
|
115
|
+
@watcher = nil
|
116
|
+
|
117
|
+
@zk.close! unless @zk.nil?
|
118
|
+
@zk = nil
|
119
|
+
|
120
|
+
log.info "synapse: zookeeper watcher cleaned up successfully"
|
121
|
+
end
|
122
|
+
|
123
|
+
def zk_connect
|
124
|
+
log.info "synapse: zookeeper watcher connecting to ZK at #{@zk_hosts}"
|
125
|
+
@zk = ZK.new(@zk_hosts)
|
126
|
+
|
127
|
+
# handle session expiry -- by cleaning up zk, this will make `ping?`
|
128
|
+
# fail and so synapse will exit
|
129
|
+
@zk.on_expired_session do
|
130
|
+
log.warn "synapse: zookeeper watcher ZK session expired!"
|
131
|
+
zk_cleanup
|
132
|
+
end
|
133
|
+
|
134
|
+
# the path must exist, otherwise watch callbacks will not work
|
135
|
+
create(@discovery['path'])
|
136
|
+
|
137
|
+
# call the callback to bootstrap the process
|
138
|
+
watcher_callback.call
|
139
|
+
end
|
140
|
+
|
141
|
+
# decode the data at a zookeeper endpoint
|
142
|
+
def deserialize_service_instance(data)
|
143
|
+
log.debug "synapse: deserializing process data"
|
144
|
+
decoded = JSON.parse(data)
|
145
|
+
|
146
|
+
host = decoded['host'] || (raise ValueError, 'instance json data does not have host key')
|
147
|
+
port = decoded['port'] || (raise ValueError, 'instance json data does not have port key')
|
148
|
+
name = decoded['name'] || nil
|
149
|
+
|
150
|
+
return host, port, name
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
#
|
3
|
+
# Variant of the Zookeeper service watcher that works with Apache Aurora
|
4
|
+
# service announcements.
|
5
|
+
#
|
6
|
+
# Parameters:
|
7
|
+
# hosts: list of zookeeper hosts to query (List of Strings, required)
|
8
|
+
# path: "/path/to/serverset/in/zookeeper" (String, required)
|
9
|
+
# port_name: Named service endpoint (String, optional)
|
10
|
+
#
|
11
|
+
# If port_name is omitted, uses the default serviceEndpoint port.
|
12
|
+
|
13
|
+
# zk node data looks like this:
|
14
|
+
#
|
15
|
+
# {
|
16
|
+
# "additionalEndpoints": {
|
17
|
+
# "aurora": {
|
18
|
+
# "host": "somehostname",
|
19
|
+
# "port": 31943
|
20
|
+
# },
|
21
|
+
# "http": {
|
22
|
+
# "host": "somehostname",
|
23
|
+
# "port": 31943
|
24
|
+
# },
|
25
|
+
# "otherport": {
|
26
|
+
# "host": "somehostname",
|
27
|
+
# "port": 31944
|
28
|
+
# }
|
29
|
+
# },
|
30
|
+
# "serviceEndpoint": {
|
31
|
+
# "host": "somehostname",
|
32
|
+
# "port": 31943
|
33
|
+
# },
|
34
|
+
# "shard": 0,
|
35
|
+
# "status": "ALIVE"
|
36
|
+
# }
|
37
|
+
#
|
38
|
+
|
39
|
+
require 'synapse/service_watcher/zookeeper'
|
40
|
+
|
41
|
+
module Synapse
|
42
|
+
# Watcher for Zookeeper announcements from Apache Aurora
|
43
|
+
class ZookeeperAuroraWatcher < Synapse::ZookeeperWatcher
|
44
|
+
def validate_discovery_opts
|
45
|
+
@discovery['method'] == 'zookeeper_aurora' ||
|
46
|
+
fail(ArgumentError,
|
47
|
+
"Invalid discovery method: #{@discovery['method']}")
|
48
|
+
@discovery['hosts'] ||
|
49
|
+
fail(ArgumentError,
|
50
|
+
"Missing or invalid zookeeper host for service #{@name}")
|
51
|
+
@discovery['path'] ||
|
52
|
+
fail(ArgumentError, "Invalid zookeeper path for service #{@name}")
|
53
|
+
end
|
54
|
+
|
55
|
+
def deserialize_service_instance(data)
|
56
|
+
log.debug 'Deserializing process data'
|
57
|
+
decoded = JSON.parse(data)
|
58
|
+
|
59
|
+
name = decoded['shard'].to_s ||
|
60
|
+
fail("Instance JSON data missing 'shard' key")
|
61
|
+
|
62
|
+
hostport = if @discovery['port_name']
|
63
|
+
decoded['additionalEndpoints'][@discovery['port_name']] ||
|
64
|
+
fail("Endpoint '#{@discovery['port_name']}' not found " \
|
65
|
+
'in instance JSON data')
|
66
|
+
else
|
67
|
+
decoded['serviceEndpoint']
|
68
|
+
end
|
69
|
+
|
70
|
+
host = hostport['host'] || fail("Instance JSON data missing 'host' key")
|
71
|
+
port = hostport['port'] || fail("Instance JSON data missing 'port' key")
|
72
|
+
|
73
|
+
[host, port, name]
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,232 @@
|
|
1
|
+
require 'synapse/service_watcher/base'
|
2
|
+
require 'synapse/service_watcher/dns'
|
3
|
+
require 'synapse/service_watcher/zookeeper'
|
4
|
+
|
5
|
+
require 'thread'
|
6
|
+
|
7
|
+
# Watcher for watching Zookeeper for entries containing DNS names that are
|
8
|
+
# continuously resolved to IP Addresses. The use case for this watcher is to
|
9
|
+
# allow services that are addressed by DNS to be reconfigured via Zookeeper
|
10
|
+
# instead of an update of the synapse config.
|
11
|
+
#
|
12
|
+
# The implementation builds on top of the existing DNS and Zookeeper watchers.
|
13
|
+
# This watcher creates a thread to manage the lifecycle of the DNS and
|
14
|
+
# Zookeeper watchers. This thread also publishes messages on a queue to
|
15
|
+
# indicate that DNS should be re-resolved (after the check interval) or that
|
16
|
+
# the DNS watcher should be shut down. The Zookeeper watcher waits for changes
|
17
|
+
# in backends from zookeeper and publishes those changes on an internal queue
|
18
|
+
# consumed by the DNS watcher. The DNS watcher blocks on this queue waiting
|
19
|
+
# for messages indicating that new servers are available, the check interval
|
20
|
+
# has passed (triggering a re-resolve), or that the watcher should shut down.
|
21
|
+
# The DNS watcher is responsible for the actual reconfiguring of backends.
|
22
|
+
module Synapse
|
23
|
+
class ZookeeperDnsWatcher < BaseWatcher
|
24
|
+
|
25
|
+
# Valid messages that can be passed through the internal message queue
|
26
|
+
module Messages
|
27
|
+
class InvalidMessageError < RuntimeError; end
|
28
|
+
|
29
|
+
# Indicates new servers identified by DNS names to be resolved. This is
|
30
|
+
# sent from Zookeeper on events that modify the ZK node. The payload is
|
31
|
+
# an array of hashes containing {'host', 'port', 'name'}
|
32
|
+
class NewServers < Struct.new(:servers); end
|
33
|
+
|
34
|
+
# Indicates that DNS should be re-resolved. This is sent by the
|
35
|
+
# ZookeeperDnsWatcher thread every check_interval seconds to cause a
|
36
|
+
# refresh of the IP addresses.
|
37
|
+
class CheckInterval; end
|
38
|
+
|
39
|
+
# Indicates that the DNS watcher should shut down. This is sent when
|
40
|
+
# stop is called.
|
41
|
+
class StopWatcher; end
|
42
|
+
|
43
|
+
# Saved instances of message types with contents that cannot vary. This
|
44
|
+
# reduces object allocation.
|
45
|
+
STOP_WATCHER_MESSAGE = StopWatcher.new
|
46
|
+
CHECK_INTERVAL_MESSAGE = CheckInterval.new
|
47
|
+
end
|
48
|
+
|
49
|
+
class Dns < Synapse::DnsWatcher
|
50
|
+
|
51
|
+
# Overrides the discovery_servers method on the parent class
|
52
|
+
attr_accessor :discovery_servers
|
53
|
+
|
54
|
+
def initialize(opts={}, synapse, message_queue)
|
55
|
+
@message_queue = message_queue
|
56
|
+
|
57
|
+
super(opts, synapse)
|
58
|
+
end
|
59
|
+
|
60
|
+
def stop
|
61
|
+
@message_queue.push(Messages::STOP_WATCHER_MESSAGE)
|
62
|
+
end
|
63
|
+
|
64
|
+
def watch
|
65
|
+
last_resolution = nil
|
66
|
+
while true
|
67
|
+
# Blocks on message queue, the message will be a signal to stop
|
68
|
+
# watching, to check a new set of servers from ZK, or to re-resolve
|
69
|
+
# the DNS (triggered every check_interval seconds)
|
70
|
+
message = @message_queue.pop
|
71
|
+
|
72
|
+
log.debug "synapse: received message #{message.inspect}"
|
73
|
+
|
74
|
+
case message
|
75
|
+
when Messages::StopWatcher
|
76
|
+
break
|
77
|
+
when Messages::NewServers
|
78
|
+
self.discovery_servers = message.servers
|
79
|
+
when Messages::CheckInterval
|
80
|
+
# Proceed to re-resolve the DNS
|
81
|
+
else
|
82
|
+
raise Messages::InvalidMessageError,
|
83
|
+
"Received unrecognized message: #{message.inspect}"
|
84
|
+
end
|
85
|
+
|
86
|
+
# Empty servers means we haven't heard back from ZK yet or ZK is
|
87
|
+
# empty. This should only occur if we don't get results from ZK
|
88
|
+
# within check_interval seconds or if ZK is empty.
|
89
|
+
if self.discovery_servers.nil? || self.discovery_servers.empty?
|
90
|
+
log.warn "synapse: no backends for service #{@name}"
|
91
|
+
else
|
92
|
+
# Resolve DNS names with the nameserver
|
93
|
+
current_resolution = resolve_servers
|
94
|
+
unless last_resolution == current_resolution
|
95
|
+
last_resolution = current_resolution
|
96
|
+
configure_backends(last_resolution)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
# Validation is skipped as it has already occurred in the parent watcher
|
105
|
+
def validate_discovery_opts
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
class Zookeeper < Synapse::ZookeeperWatcher
|
110
|
+
def initialize(opts={}, synapse, message_queue)
|
111
|
+
super(opts, synapse)
|
112
|
+
|
113
|
+
@message_queue = message_queue
|
114
|
+
end
|
115
|
+
|
116
|
+
# Overrides reconfigure! to cause the new list of servers to be messaged
|
117
|
+
# to the DNS watcher rather than invoking a synapse reconfigure directly
|
118
|
+
def reconfigure!
|
119
|
+
# push the new backends onto the queue
|
120
|
+
@message_queue.push(Messages::NewServers.new(@backends))
|
121
|
+
end
|
122
|
+
|
123
|
+
private
|
124
|
+
|
125
|
+
# Validation is skipped as it has already occurred in the parent watcher
|
126
|
+
def validate_discovery_opts
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def start
|
131
|
+
dns_discovery_opts = @discovery.select do |k,_|
|
132
|
+
k == 'nameserver'
|
133
|
+
end
|
134
|
+
|
135
|
+
zookeeper_discovery_opts = @discovery.select do |k,_|
|
136
|
+
k == 'hosts' || k == 'path'
|
137
|
+
end
|
138
|
+
|
139
|
+
@check_interval = @discovery['check_interval'] || 30.0
|
140
|
+
|
141
|
+
@message_queue = Queue.new
|
142
|
+
|
143
|
+
@dns = Dns.new(
|
144
|
+
mk_child_watcher_opts(dns_discovery_opts),
|
145
|
+
@synapse,
|
146
|
+
@message_queue
|
147
|
+
)
|
148
|
+
|
149
|
+
@zk = Zookeeper.new(
|
150
|
+
mk_child_watcher_opts(zookeeper_discovery_opts),
|
151
|
+
@synapse,
|
152
|
+
@message_queue
|
153
|
+
)
|
154
|
+
|
155
|
+
@zk.start
|
156
|
+
@dns.start
|
157
|
+
|
158
|
+
@watcher = Thread.new do
|
159
|
+
until @should_exit
|
160
|
+
# Trigger a DNS resolve every @check_interval seconds
|
161
|
+
sleep @check_interval
|
162
|
+
|
163
|
+
# Only trigger the resolve if the queue is empty, every other message
|
164
|
+
# on the queue would either cause a resolve or stop the watcher
|
165
|
+
if @message_queue.empty?
|
166
|
+
@message_queue.push(Messages::CHECK_INTERVAL_MESSAGE)
|
167
|
+
end
|
168
|
+
|
169
|
+
end
|
170
|
+
log.info "synapse: zookeeper_dns watcher exited successfully"
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
def ping?
|
175
|
+
@watcher.alive? && @dns.ping? && @zk.ping?
|
176
|
+
end
|
177
|
+
|
178
|
+
def stop
|
179
|
+
super
|
180
|
+
|
181
|
+
@dns.stop
|
182
|
+
@zk.stop
|
183
|
+
end
|
184
|
+
|
185
|
+
def backends
|
186
|
+
@dns.backends
|
187
|
+
end
|
188
|
+
|
189
|
+
private
|
190
|
+
|
191
|
+
def validate_discovery_opts
|
192
|
+
unless @discovery['method'] == 'zookeeper_dns'
|
193
|
+
raise ArgumentError, "invalid discovery method #{@discovery['method']}"
|
194
|
+
end
|
195
|
+
|
196
|
+
unless @discovery['hosts']
|
197
|
+
raise ArgumentError, "missing or invalid zookeeper host for service #{@name}"
|
198
|
+
end
|
199
|
+
|
200
|
+
unless @discovery['path']
|
201
|
+
raise ArgumentError, "invalid zookeeper path for service #{@name}"
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
# Method to generate a full config for the children (Dns and Zookeeper)
|
206
|
+
# watchers
|
207
|
+
#
|
208
|
+
# Notes on passing in the default_servers:
|
209
|
+
#
|
210
|
+
# Setting the default_servers here allows the Zookeeper watcher to return
|
211
|
+
# a list of backends based on the default servers when it fails to find
|
212
|
+
# any matching servers. These are passed on as the discovered backends
|
213
|
+
# to the DNS watcher, which will then watch them as normal for DNS
|
214
|
+
# changes. The default servers can also come into play if none of the
|
215
|
+
# hostnames from Zookeeper resolve to addresses in the DNS watcher. This
|
216
|
+
# should generally result in the expected behavior, but caution should be
|
217
|
+
# taken when deciding that this is the desired behavior.
|
218
|
+
def mk_child_watcher_opts(discovery_opts)
|
219
|
+
{
|
220
|
+
'name' => @name,
|
221
|
+
'haproxy' => @haproxy,
|
222
|
+
'discovery' => discovery_opts,
|
223
|
+
'default_servers' => @default_servers,
|
224
|
+
}
|
225
|
+
end
|
226
|
+
|
227
|
+
# Override reconfigure! as this class should not explicitly reconfigure
|
228
|
+
# synapse
|
229
|
+
def reconfigure!
|
230
|
+
end
|
231
|
+
end
|
232
|
+
end
|