em-voldemort 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,247 @@
1
+ module EM::Voldemort
2
+ # A client for a Voldemort cluster. The cluster is initialized by giving the hostname and port of
3
+ # one of its nodes, and the other nodes are autodiscovered.
4
+ #
5
+ # TODO if one node is down, a request should be retried on a replica.
6
+ class Cluster
7
+ include Protocol
8
+
9
+ attr_reader :bootstrap_host, :bootstrap_port, :logger, :cluster_name
10
+
11
+ RETRY_BOOTSTRAP_PERIOD = 10 # seconds
12
+ METADATA_STORE_NAME = 'metadata'.freeze
13
+ CLUSTER_INFO_KEY = 'cluster.xml'.freeze
14
+ STORES_INFO_KEY = 'stores.xml'.freeze
15
+
16
+ def initialize(options={})
17
+ @bootstrap_host = options[:host] or raise "#{self.class.name} requires :host"
18
+ @bootstrap_port = options[:port] or raise "#{self.class.name} requires :port"
19
+ @logger = options[:logger] || Logger.new($stdout)
20
+ @bootstrap_state = :not_started
21
+ @stores = {}
22
+ end
23
+
24
+ # Bootstraps the cluster (discovers all cluster nodes and metadata by connecting to the one node
25
+ # that was specified). Calling #connect is optional, since it also happens automatically when
26
+ # you start making requests.
27
+ def connect
28
+ @bootstrap_timer ||= setup_bootstrap_timer do
29
+ start_bootstrap if @bootstrap_state == :not_started || @bootstrap_state == :failed
30
+ end
31
+ start_bootstrap if @bootstrap_state == :not_started
32
+ @bootstrap
33
+ end
34
+
35
+ # Fetches the value associated with a particular key in a particular store. Returns a deferrable
36
+ # that succeeds with the value, or fails with an exception object.
37
+ def get(store_name, key, router=nil)
38
+ when_ready do |deferrable|
39
+ connections = choose_connections(key, router)
40
+ if connections.size == 0
41
+ deferrable.fail(ServerError.new('No connection can handle the request'))
42
+ elsif connections.first.health == :good
43
+ # Send the request to the preferred node, but fall back to others if it fails.
44
+ get_in_sequence(connections, store_name, key, deferrable)
45
+ else
46
+ # The request to the first node will probably fail, but we send it anyway, so that the
47
+ # connection can discover when the node comes back up. Make the request to other
48
+ # connections at the same time to avoid delaying the request.
49
+ get_in_parallel(connections, store_name, key, deferrable)
50
+ end
51
+ end
52
+ end
53
+
54
+ # Returns a {Store} object configured for accessing a particular store on the cluster.
55
+ def store(store_name)
56
+ @stores[store_name.to_s] ||= Store.new(self, store_name)
57
+ end
58
+
59
+ private
60
+
61
+ def setup_bootstrap_timer
62
+ EM.add_periodic_timer(RETRY_BOOTSTRAP_PERIOD) { yield }
63
+ end
64
+
65
+ def start_bootstrap
66
+ @bootstrap_state = :started
67
+ @bootstrap_conn = Connection.new(:host => bootstrap_host, :port => bootstrap_port, :logger => logger)
68
+ @bootstrap = EM::DefaultDeferrable.new
69
+
70
+ cluster_req = get_from_connection(@bootstrap_conn, METADATA_STORE_NAME, CLUSTER_INFO_KEY)
71
+
72
+ cluster_req.callback do |cluster_xml|
73
+ if parse_cluster_info(cluster_xml) == :cluster_info_ok
74
+ stores_req = get_from_connection(@bootstrap_conn, METADATA_STORE_NAME, STORES_INFO_KEY)
75
+ stores_req.callback do |stores_xml|
76
+ parse_stores_info(stores_xml)
77
+ finish_bootstrap
78
+ end
79
+ stores_req.errback do |error|
80
+ logger.warn "Could not load Voldemort's stores.xml: #{error}"
81
+ @bootstrap_state = :failed
82
+ finish_bootstrap
83
+ end
84
+ else
85
+ finish_bootstrap
86
+ end
87
+ end
88
+
89
+ cluster_req.errback do |error|
90
+ logger.warn "Could not load Voldemort's cluster.xml: #{error}"
91
+ @bootstrap_state = :failed
92
+ finish_bootstrap
93
+ end
94
+ end
95
+
96
+ def finish_bootstrap
97
+ @bootstrap_conn.close
98
+ @bootstrap_conn = nil
99
+ deferrable = @bootstrap
100
+ @bootstrap = nil
101
+ if @bootstrap_state == :complete
102
+ @bootstrap_timer.cancel
103
+ @bootstrap_timer = nil
104
+ deferrable.succeed
105
+ else
106
+ deferrable.fail
107
+ end
108
+ end
109
+
110
+ # Delays execution of the block until bootstrap has completed. Returns a new deferrable, and
111
+ # passes the same deferrable to the block when it is executed (it's up to the block to make the
112
+ # deferrable succeed or fail).
113
+ def when_ready(&block)
114
+ connect
115
+ request = EM::DefaultDeferrable.new
116
+ case @bootstrap_state
117
+ when :started, :cluster_info_ok
118
+ @bootstrap.callback { yield request }
119
+ @bootstrap.errback { request.fail(ServerError.new('Could not bootstrap Voldemort cluster')) }
120
+ when :complete
121
+ yield request
122
+ when :failed
123
+ request.fail(ServerError.new('Could not bootstrap Voldemort cluster'))
124
+ else
125
+ raise "bad bootstrap_state: #{@bootstrap_state.inspect}"
126
+ end
127
+ request
128
+ end
129
+
130
+ # Parses Voldemort's cluster.xml configuration file, as obtained in the bootstrap process.
131
+ def parse_cluster_info(cluster_xml)
132
+ doc = Nokogiri::XML(cluster_xml)
133
+ @cluster_name = doc.xpath('/cluster/name').text
134
+ @node_by_id = {}
135
+ @node_by_partition = {}
136
+ doc.xpath('/cluster/server').each do |node|
137
+ node_id = node.xpath('id').text.to_i
138
+ connection = Connection.new(
139
+ :host => node.xpath('host').text,
140
+ :port => node.xpath('socket-port').text.to_i,
141
+ :node_id => node_id,
142
+ :logger => logger
143
+ )
144
+ @node_by_id[node_id] = connection
145
+ node.xpath('partitions').text.split(/\s*,\s*/).map(&:to_i).each do |partition|
146
+ raise "duplicate assignment of partition #{partition}" if @node_by_partition[partition]
147
+ @node_by_partition[partition] = connection
148
+ end
149
+ end
150
+ raise 'no partitions defined on cluster' if @node_by_partition.empty?
151
+ (0...@node_by_partition.size).each do |partition|
152
+ raise "missing node assignment for partition #{partition}" unless @node_by_partition[partition]
153
+ end
154
+ @bootstrap_state = :cluster_info_ok
155
+ rescue => e
156
+ logger.warn "Error processing cluster.xml: #{e}"
157
+ @bootstrap_state = :failed
158
+ end
159
+
160
+ def parse_stores_info(stores_xml)
161
+ doc = Nokogiri::XML(stores_xml)
162
+ doc.xpath('/stores/store').each do |store|
163
+ store_name = store.xpath('name').text
164
+ @stores[store_name] ||= Store.new(self, store_name)
165
+ @stores[store_name].load_config(store)
166
+ end
167
+ @bootstrap_state = :complete
168
+ rescue => e
169
+ logger.warn "Error processing stores.xml: #{e}"
170
+ @bootstrap_state = :failed
171
+ end
172
+
173
+ def choose_connections(key, router=nil)
174
+ if router
175
+ router.partitions(key, @node_by_partition).map do |partition|
176
+ @node_by_partition[partition]
177
+ end.compact
178
+ else
179
+ @node_by_id.values.sample(2) # choose two random connections
180
+ end
181
+ end
182
+
183
+ # Makes a 'get' request to the first connection in the given list of connections. If the request
184
+ # fails with a ServerError, retries the request on the next connection in the list, etc,
185
+ # eventually failing if none of the connections can successfully handle the request.
186
+ def get_in_sequence(connections, store_name, key, deferrable)
187
+ raise ArgumentError, 'connections must not be empty' if connections.empty?
188
+ request = get_from_connection(connections.first, store_name, key)
189
+ request.callback {|response| deferrable.succeed(response) }
190
+ request.errback do |error|
191
+ if error.is_a?(ServerError) && connections.size > 1
192
+ get_in_sequence(connections.drop(1), store_name, key, deferrable)
193
+ else
194
+ deferrable.fail(error)
195
+ end
196
+ end
197
+ end
198
+
199
+ # Makes a 'get' request to all given connections in parallel. Succeeds with the first successful
200
+ # response, or fails if all connections' requests fail.
201
+ def get_in_parallel(connections, store_name, key, deferrable)
202
+ raise ArgumentError, 'connections must not be empty' if connections.empty?
203
+ responses = 0
204
+ done = false
205
+ connections.each do |connection|
206
+ request = get_from_connection(connection, store_name, key)
207
+ request.callback do |response|
208
+ deferrable.succeed(response) unless done
209
+ done = true
210
+ end
211
+ request.errback do |error|
212
+ if error.is_a?(ClientError) && !done
213
+ deferrable.fail(error)
214
+ done = true
215
+ elsif !done
216
+ responses += 1
217
+ deferrable.fail(error) if responses == connections.size
218
+ end
219
+ end
220
+ end
221
+ end
222
+
223
+ # Makes a 'get' request for a particular key to a particular Voldemort store, using a particular
224
+ # connection. Returns a deferrable that succeeds with the value in the store if successful, or
225
+ # fails with an exception object if not.
226
+ def get_from_connection(connection, store_name, key, deferrable=EM::DefaultDeferrable.new)
227
+ request = connection.send_request(get_request(store_name, key))
228
+
229
+ request.callback do |response|
230
+ begin
231
+ parsed_response = get_response(response)
232
+ rescue ClientError => error
233
+ deferrable.fail(error)
234
+ rescue => error
235
+ message = "protocol error #{error.class}: #{error.message} while parsing response: #{response.inspect}"
236
+ logger.error(message)
237
+ deferrable.fail(ServerError.new(message))
238
+ else
239
+ deferrable.succeed(parsed_response)
240
+ end
241
+ end
242
+
243
+ request.errback {|response| deferrable.fail(response) }
244
+ deferrable
245
+ end
246
+ end
247
+ end
@@ -0,0 +1,39 @@
1
+ module EM::Voldemort
2
+ # Compression/decompression codec for keys and values in a store
3
+ class Compressor
4
+
5
+ attr_reader :type, :options
6
+
7
+ def initialize(xml)
8
+ @type = xml && xml.xpath('type').text
9
+ @options = xml && xml.xpath('options').text
10
+ if type != nil && type != 'gzip'
11
+ raise "Unsupported compression codec: #{type}"
12
+ end
13
+ end
14
+
15
+ def encode(data)
16
+ case type
17
+ when nil
18
+ data
19
+ when 'gzip'
20
+ buffer = StringIO.new
21
+ buffer.set_encoding(Encoding::BINARY)
22
+ gz = Zlib::GzipWriter.new(buffer)
23
+ gz.write(data)
24
+ gz.close
25
+ buffer.rewind
26
+ buffer.string
27
+ end
28
+ end
29
+
30
+ def decode(data)
31
+ case type
32
+ when nil
33
+ data
34
+ when 'gzip'
35
+ Zlib::GzipReader.new(StringIO.new(data)).read.force_encoding(Encoding::BINARY)
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,234 @@
1
+ module EM::Voldemort
2
+ # TCP connection to one Voldemort node. The connection can be used to access multiple stores.
3
+ # Automatically reconnects if the connection is lost, but does not automatically retry failed
4
+ # requests (that is the cluster's job).
5
+ class Connection
6
+ attr_reader :host, :port, :node_id, :protocol, :logger, :health
7
+
8
+ DEFAULT_PROTOCOL = 'pb0' # Voldemort's protobuf-based protocol
9
+ STATUS_CHECK_PERIOD = 5 # Every 5 seconds, check on the health of the connection
10
+ REQUEST_TIMEOUT = 5 # If a request takes longer than 5 seconds, close the connection
11
+
12
+ def initialize(options={})
13
+ @host = options[:host] or raise ArgumentError, "#{self.class.name} requires :host"
14
+ @port = options[:port] or raise ArgumentError, "#{self.class.name} requires :port"
15
+ @node_id = options[:node_id]
16
+ @protocol = options[:protocol] || DEFAULT_PROTOCOL
17
+ @logger = options[:logger] || Logger.new($stdout)
18
+ @health = :good
19
+ end
20
+
21
+ # Establishes a connection to the node. Calling #connect is optional, since it also happens
22
+ # automatically when you start making requests.
23
+ def connect
24
+ force_connect unless @handler
25
+ end
26
+
27
+ # Sends a request to the node, given as a binary string (not including the request size prefix).
28
+ # Establishes a connection if necessary. If a request is already in progress, this request is
29
+ # queued up. Returns a deferrable that succeeds with the node's response (again without the size
30
+ # prefix), or fails if there was a network-level error.
31
+ def send_request(request)
32
+ connect
33
+ @handler.enqueue_request(request)
34
+ end
35
+
36
+ # Waits for the outstanding request (if any) to complete, then gracefully shuts down the
37
+ # connection. Returns a deferrable that succeeds once the connection is closed (never fails).
38
+ def close
39
+ return @closing_deferrable if @closing_deferrable
40
+ @closing_deferrable = EM::DefaultDeferrable.new
41
+ @timer.cancel
42
+
43
+ if @handler
44
+ @handler.close_gracefully
45
+ else
46
+ @closing_deferrable.succeed
47
+ end
48
+
49
+ @handler = FailHandler.new(self)
50
+ @health = :bad
51
+ @closing_deferrable
52
+ end
53
+
54
+ # Called by the connection handler when the connection is closed for any reason (closed by us,
55
+ # closed by peer, rejected, timeout etc). Do not call from application code.
56
+ def connection_closed(handler, reason=nil)
57
+ logger.info ["Connection to Voldemort node #{host}:#{port} closed", reason].compact.join(': ')
58
+ @handler = FailHandler.new(self) if handler.equal? @handler
59
+ @health = :bad
60
+ @closing_deferrable.succeed if @closing_deferrable
61
+ end
62
+
63
+ private
64
+
65
+ def setup_status_check_timer
66
+ EM.add_periodic_timer(STATUS_CHECK_PERIOD) { yield }
67
+ end
68
+
69
+ def status_check
70
+ if @closing_deferrable
71
+ # Do nothing (don't reconnect once we've been asked to shut down).
72
+ elsif !@handler || @handler.is_a?(FailHandler)
73
+ # Connect for the first time, or reconnect after failure.
74
+ force_connect
75
+ elsif @handler.in_flight && Time.now - @handler.last_request >= REQUEST_TIMEOUT
76
+ # Request timed out. Pronounce the connection dead, and reconnect.
77
+ @handler.close_connection
78
+ force_connect
79
+ end
80
+ end
81
+
82
+ def force_connect
83
+ @timer ||= setup_status_check_timer(&method(:status_check))
84
+ @handler = EM.connect(host, port, Handler, self)
85
+ @handler.in_flight.callback { @health = :good } # restore health if protocol negotiation succeeded
86
+ rescue EventMachine::ConnectionError => e
87
+ # A synchronous exception is typically thrown on DNS resolution failure
88
+ logger.warn "Cannot connect to Voldemort node: #{e.class.name}: #{e.message}"
89
+ connection_closed(@handler)
90
+ @handler = FailHandler.new(self)
91
+ end
92
+
93
+
94
+ # EventMachine handler for a Voldemort node connection
95
+ module Handler
96
+ # The EM::Voldemort::Connection object for which we're handling the connection
97
+ attr_reader :connection
98
+
99
+ # State machine. One of :connecting, :protocol_proposal, :idle, :request, :disconnected
100
+ attr_reader :state
101
+
102
+ # If a request is currently in flight, this is a deferrable that will succeed or fail when the
103
+ # request completes. The protocol requires that only one request can be in flight at once.
104
+ attr_reader :in_flight
105
+
106
+ # The time at which the request currently in flight was sent
107
+ attr_reader :last_request
108
+
109
+ # Array of [request_data, deferrable] pairs, containing requests that have not yet been sent
110
+ attr_reader :request_queue
111
+
112
+ def initialize(connection)
113
+ @connection = connection
114
+ @state = :connecting
115
+ @in_flight = EM::DefaultDeferrable.new
116
+ @last_request = Time.now
117
+ @request_queue = []
118
+ end
119
+
120
+ def enqueue_request(request)
121
+ EM::DefaultDeferrable.new.tap do |deferrable|
122
+ request_queue << [request, deferrable]
123
+ send_next_request unless in_flight
124
+ end
125
+ end
126
+
127
+ # First action when the connection is established: client tells the server which version of
128
+ # the Voldemort protocol it wants to use
129
+ def send_protocol_proposal(protocol)
130
+ raise ArgumentError, 'protocol must be 3 bytes long' if protocol.bytesize != 3
131
+ raise "unexpected state before protocol proposal: #{@state.inspect}" unless @state == :connecting
132
+ send_data(protocol)
133
+ @state = :protocol_proposal
134
+ end
135
+
136
+ # Takes the request at the front of the queue and sends it to the Voldemort node
137
+ def send_next_request
138
+ return if request_queue.empty?
139
+ raise "cannot make a request while in #{@state.inspect} state" unless @state == :idle
140
+ request, @in_flight = request_queue.shift
141
+ send_data([request.size, request].pack('NA*'))
142
+ @recv_buf = ''.force_encoding('BINARY')
143
+ @last_request = Time.now
144
+ @state = :request
145
+ end
146
+
147
+ # Connection established (called by EventMachine)
148
+ def post_init
149
+ connection.logger.info "Connected to Voldemort node at #{connection.host}:#{connection.port}"
150
+ send_protocol_proposal(connection.protocol)
151
+ in_flight.errback do |response|
152
+ connection.logger.warn "Voldemort protocol #{connection.protocol} not accepted: #{response.inspect}"
153
+ end
154
+ end
155
+
156
+ # The Voldemort node is talking to us (called by EventMachine)
157
+ def receive_data(data)
158
+ case @state
159
+ when :protocol_proposal
160
+ deferrable = @in_flight
161
+ @state = :idle
162
+ @in_flight = nil
163
+ if data == 'ok'
164
+ send_next_request
165
+ deferrable.succeed
166
+ else
167
+ close_connection
168
+ deferrable.fail("server response: #{data.inspect}")
169
+ end
170
+
171
+ when :request
172
+ @recv_buf << data
173
+ response_size = @recv_buf.unpack('N').first
174
+ if response_size && @recv_buf.bytesize >= response_size + 4
175
+ response = @recv_buf[4, response_size]
176
+ deferrable = @in_flight
177
+ @state = :idle
178
+ @in_flight = @recv_buf = nil
179
+ send_next_request
180
+ deferrable.succeed(response)
181
+ end
182
+
183
+ else
184
+ raise "Received data in unexpected state: #{@state.inspect}"
185
+ end
186
+ end
187
+
188
+ # Connection is asking us to shut down. Wait for the currently in-flight request to complete,
189
+ # but fail any unsent requests in the queue.
190
+ def close_gracefully
191
+ @request_queue.each {|request, deferrable| deferrable.fail(ServerError.new('shutdown requested')) }
192
+ @request_queue = []
193
+ if in_flight
194
+ in_flight.callback { close_connection }
195
+ in_flight.errback { close_connection }
196
+ else
197
+ close_connection
198
+ end
199
+ end
200
+
201
+ # Connection closed (called by EventMachine)
202
+ def unbind(reason=nil)
203
+ @state = :disconnected
204
+ deferrable = @in_flight
205
+ @in_flight = nil
206
+ deferrable.fail(ServerError.new('connection closed')) if deferrable
207
+ @request_queue.each {|request, deferrable| deferrable.fail(ServerError.new('connection closed')) }
208
+ @request_queue = []
209
+ connection.connection_closed(self, reason)
210
+ end
211
+ end
212
+
213
+
214
+ # Quacks like a EM::Voldemort::Connection::Handler, but fails all requests.
215
+ # Useful for representing a connection in an error state.
216
+ class FailHandler
217
+ attr_reader :in_flight
218
+
219
+ def initialize(connection)
220
+ @connection = connection
221
+ end
222
+
223
+ def enqueue_request(request)
224
+ EM::DefaultDeferrable.new.tap do |deferrable|
225
+ deferrable.fail(ServerError.new('Connection to Voldemort node closed'))
226
+ end
227
+ end
228
+
229
+ def close_gracefully
230
+ @connection.connection_closed(self)
231
+ end
232
+ end
233
+ end
234
+ end