dalli 3.0.4 → 3.1.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of dalli might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Gemfile +11 -5
- data/History.md +31 -0
- data/README.md +25 -134
- data/lib/dalli/cas/client.rb +2 -0
- data/lib/dalli/client.rb +215 -323
- data/lib/dalli/compressor.rb +13 -4
- data/lib/dalli/key_manager.rb +113 -0
- data/lib/dalli/options.rb +5 -5
- data/lib/dalli/pipelined_getter.rb +177 -0
- data/lib/dalli/protocol/binary/request_formatter.rb +117 -0
- data/lib/dalli/protocol/binary/response_header.rb +36 -0
- data/lib/dalli/protocol/binary/response_processor.rb +200 -0
- data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
- data/lib/dalli/protocol/binary.rb +251 -561
- data/lib/dalli/protocol/connection_manager.rb +242 -0
- data/lib/dalli/protocol/response_buffer.rb +53 -0
- data/lib/dalli/protocol/server_config_parser.rb +22 -5
- data/lib/dalli/protocol/value_marshaller.rb +59 -0
- data/lib/dalli/protocol/value_serializer.rb +91 -0
- data/lib/dalli/protocol.rb +2 -3
- data/lib/dalli/ring.rb +95 -35
- data/lib/dalli/server.rb +2 -2
- data/lib/dalli/servers_arg_normalizer.rb +54 -0
- data/lib/dalli/socket.rb +101 -55
- data/lib/dalli/version.rb +3 -1
- data/lib/dalli.rb +39 -14
- data/lib/rack/session/dalli.rb +95 -76
- metadata +80 -6
@@ -0,0 +1,242 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'English'
|
4
|
+
require 'socket'
|
5
|
+
require 'timeout'
|
6
|
+
|
7
|
+
module Dalli
|
8
|
+
module Protocol
|
9
|
+
##
|
10
|
+
# Manages the socket connection to the server, including ensuring liveness
|
11
|
+
# and retries.
|
12
|
+
##
|
13
|
+
class ConnectionManager
|
14
|
+
DEFAULTS = {
|
15
|
+
# seconds between trying to contact a remote server
|
16
|
+
down_retry_delay: 30,
|
17
|
+
# connect/read/write timeout for socket operations
|
18
|
+
socket_timeout: 1,
|
19
|
+
# times a socket operation may fail before considering the server dead
|
20
|
+
socket_max_failures: 2,
|
21
|
+
# amount of time to sleep between retries when a failure occurs
|
22
|
+
socket_failure_delay: 0.1,
|
23
|
+
# Set keepalive
|
24
|
+
keepalive: true
|
25
|
+
}.freeze
|
26
|
+
|
27
|
+
attr_accessor :hostname, :port, :socket_type, :options
|
28
|
+
attr_reader :sock
|
29
|
+
|
30
|
+
def initialize(hostname, port, socket_type, client_options)
|
31
|
+
@hostname = hostname
|
32
|
+
@port = port
|
33
|
+
@socket_type = socket_type
|
34
|
+
@options = DEFAULTS.merge(client_options)
|
35
|
+
@request_in_progress = false
|
36
|
+
@sock = nil
|
37
|
+
@pid = nil
|
38
|
+
|
39
|
+
reset_down_info
|
40
|
+
end
|
41
|
+
|
42
|
+
def name
|
43
|
+
if socket_type == :unix
|
44
|
+
hostname
|
45
|
+
else
|
46
|
+
"#{hostname}:#{port}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def establish_connection
|
51
|
+
Dalli.logger.debug { "Dalli::Server#connect #{name}" }
|
52
|
+
|
53
|
+
@sock = memcached_socket
|
54
|
+
@pid = Process.pid
|
55
|
+
rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
|
56
|
+
# SocketError = DNS resolution failure
|
57
|
+
error_on_request!(e)
|
58
|
+
end
|
59
|
+
|
60
|
+
def reconnect_down_server?
|
61
|
+
return true unless @last_down_at
|
62
|
+
|
63
|
+
time_to_next_reconnect = @last_down_at + options[:down_retry_delay] - Time.now
|
64
|
+
return true unless time_to_next_reconnect.positive?
|
65
|
+
|
66
|
+
Dalli.logger.debug do
|
67
|
+
format('down_retry_delay not reached for %<name>s (%<time>.3f seconds left)', name: name,
|
68
|
+
time: time_to_next_reconnect)
|
69
|
+
end
|
70
|
+
false
|
71
|
+
end
|
72
|
+
|
73
|
+
def up!
|
74
|
+
log_up_detected
|
75
|
+
reset_down_info
|
76
|
+
end
|
77
|
+
|
78
|
+
# Marks the server instance as down. Updates the down_at state
|
79
|
+
# and raises an Dalli::NetworkError that includes the underlying
|
80
|
+
# error in the message. Calls close to clean up socket state
|
81
|
+
def down!
|
82
|
+
close
|
83
|
+
log_down_detected
|
84
|
+
|
85
|
+
@error = $ERROR_INFO&.class&.name
|
86
|
+
@msg ||= $ERROR_INFO&.message
|
87
|
+
raise_down_error
|
88
|
+
end
|
89
|
+
|
90
|
+
def raise_down_error
|
91
|
+
raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
|
92
|
+
end
|
93
|
+
|
94
|
+
def socket_timeout
|
95
|
+
@socket_timeout ||= @options[:socket_timeout]
|
96
|
+
end
|
97
|
+
|
98
|
+
def confirm_ready!
|
99
|
+
error_on_request!(RuntimeError.new('Already writing to socket')) if request_in_progress?
|
100
|
+
close_on_fork if fork_detected?
|
101
|
+
end
|
102
|
+
|
103
|
+
def close
|
104
|
+
return unless @sock
|
105
|
+
|
106
|
+
begin
|
107
|
+
@sock.close
|
108
|
+
rescue StandardError
|
109
|
+
nil
|
110
|
+
end
|
111
|
+
@sock = nil
|
112
|
+
@pid = nil
|
113
|
+
abort_request!
|
114
|
+
end
|
115
|
+
|
116
|
+
def connected?
|
117
|
+
!@sock.nil?
|
118
|
+
end
|
119
|
+
|
120
|
+
def request_in_progress?
|
121
|
+
@request_in_progress
|
122
|
+
end
|
123
|
+
|
124
|
+
def start_request!
|
125
|
+
@request_in_progress = true
|
126
|
+
end
|
127
|
+
|
128
|
+
def finish_request!
|
129
|
+
@request_in_progress = false
|
130
|
+
end
|
131
|
+
|
132
|
+
def abort_request!
|
133
|
+
@request_in_progress = false
|
134
|
+
end
|
135
|
+
|
136
|
+
def read(count)
|
137
|
+
start_request!
|
138
|
+
data = @sock.readfull(count)
|
139
|
+
finish_request!
|
140
|
+
data
|
141
|
+
rescue SystemCallError, Timeout::Error, EOFError => e
|
142
|
+
error_on_request!(e)
|
143
|
+
end
|
144
|
+
|
145
|
+
def write(bytes)
|
146
|
+
start_request!
|
147
|
+
result = @sock.write(bytes)
|
148
|
+
finish_request!
|
149
|
+
result
|
150
|
+
rescue SystemCallError, Timeout::Error => e
|
151
|
+
error_on_request!(e)
|
152
|
+
end
|
153
|
+
|
154
|
+
# Non-blocking read. Should only be used in the context
|
155
|
+
# of a caller who has called start_request!, but not yet
|
156
|
+
# called finish_request!. Here to support the operation
|
157
|
+
# of the get_multi operation
|
158
|
+
def read_nonblock
|
159
|
+
@sock.read_available
|
160
|
+
end
|
161
|
+
|
162
|
+
def max_allowed_failures
|
163
|
+
@max_allowed_failures ||= @options[:socket_max_failures] || 2
|
164
|
+
end
|
165
|
+
|
166
|
+
def error_on_request!(err_or_string)
|
167
|
+
log_warn_message(err_or_string)
|
168
|
+
|
169
|
+
@fail_count += 1
|
170
|
+
if @fail_count >= max_allowed_failures
|
171
|
+
down!
|
172
|
+
else
|
173
|
+
# Closes the existing socket, setting up for a reconnect
|
174
|
+
# on next request
|
175
|
+
reconnect!('Socket operation failed, retrying...')
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
def reconnect!(message)
|
180
|
+
close
|
181
|
+
sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
|
182
|
+
raise Dalli::NetworkError, message
|
183
|
+
end
|
184
|
+
|
185
|
+
def reset_down_info
|
186
|
+
@fail_count = 0
|
187
|
+
@down_at = nil
|
188
|
+
@last_down_at = nil
|
189
|
+
@msg = nil
|
190
|
+
@error = nil
|
191
|
+
end
|
192
|
+
|
193
|
+
def memcached_socket
|
194
|
+
if socket_type == :unix
|
195
|
+
Dalli::Socket::UNIX.open(hostname, options)
|
196
|
+
else
|
197
|
+
Dalli::Socket::TCP.open(hostname, port, options)
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
def log_warn_message(err_or_string)
|
202
|
+
detail = err_or_string.is_a?(String) ? err_or_string : "#{err_or_string.class}: #{err_or_string.message}"
|
203
|
+
Dalli.logger.warn do
|
204
|
+
detail = err_or_string.is_a?(String) ? err_or_string : "#{err_or_string.class}: #{err_or_string.message}"
|
205
|
+
"#{name} failed (count: #{@fail_count}) #{detail}"
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def close_on_fork
|
210
|
+
message = 'Fork detected, re-connecting child process...'
|
211
|
+
Dalli.logger.info { message }
|
212
|
+
# Close socket on a fork, setting us up for reconnect
|
213
|
+
# on next request.
|
214
|
+
close
|
215
|
+
raise Dalli::NetworkError, message
|
216
|
+
end
|
217
|
+
|
218
|
+
def fork_detected?
|
219
|
+
@pid && @pid != Process.pid
|
220
|
+
end
|
221
|
+
|
222
|
+
def log_down_detected
|
223
|
+
@last_down_at = Time.now
|
224
|
+
|
225
|
+
if @down_at
|
226
|
+
time = Time.now - @down_at
|
227
|
+
Dalli.logger.debug { format('%<name>s is still down (for %<time>.3f seconds now)', name: name, time: time) }
|
228
|
+
else
|
229
|
+
@down_at = @last_down_at
|
230
|
+
Dalli.logger.warn("#{name} is down")
|
231
|
+
end
|
232
|
+
end
|
233
|
+
|
234
|
+
def log_up_detected
|
235
|
+
return unless @down_at
|
236
|
+
|
237
|
+
time = Time.now - @down_at
|
238
|
+
Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'socket'
|
4
|
+
require 'timeout'
|
5
|
+
|
6
|
+
module Dalli
|
7
|
+
module Protocol
|
8
|
+
##
|
9
|
+
# Manages the buffer for responses from memcached.
|
10
|
+
##
|
11
|
+
class ResponseBuffer
|
12
|
+
def initialize(io_source, response_processor)
|
13
|
+
@io_source = io_source
|
14
|
+
@response_processor = response_processor
|
15
|
+
end
|
16
|
+
|
17
|
+
def read
|
18
|
+
@buffer << @io_source.read_nonblock
|
19
|
+
end
|
20
|
+
|
21
|
+
# Attempts to process a single response from the buffer. Starts
|
22
|
+
# by advancing the buffer to the specified start position
|
23
|
+
def process_single_getk_response
|
24
|
+
bytes, resp_header, key, value = @response_processor.getk_response_from_buffer(@buffer)
|
25
|
+
advance(bytes)
|
26
|
+
[resp_header, key, value]
|
27
|
+
end
|
28
|
+
|
29
|
+
# Advances the internal response buffer by bytes_to_advance
|
30
|
+
# bytes. The
|
31
|
+
def advance(bytes_to_advance)
|
32
|
+
return unless bytes_to_advance.positive?
|
33
|
+
|
34
|
+
@buffer = @buffer[bytes_to_advance..-1]
|
35
|
+
end
|
36
|
+
|
37
|
+
# Resets the internal buffer to an empty state,
|
38
|
+
# so that we're ready to read pipelined responses
|
39
|
+
def reset
|
40
|
+
@buffer = +''
|
41
|
+
end
|
42
|
+
|
43
|
+
# Clear the internal response buffer
|
44
|
+
def clear
|
45
|
+
@buffer = nil
|
46
|
+
end
|
47
|
+
|
48
|
+
def in_progress?
|
49
|
+
!@buffer.nil?
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -8,19 +8,36 @@ module Dalli
|
|
8
8
|
# socket_type.
|
9
9
|
##
|
10
10
|
class ServerConfigParser
|
11
|
+
MEMCACHED_URI_PROTOCOL = 'memcached://'
|
12
|
+
|
11
13
|
# TODO: Revisit this, especially the IP/domain part. Likely
|
12
14
|
# can limit character set to LDH + '.'. Hex digit section
|
13
|
-
#
|
14
|
-
#
|
15
|
+
# is there to support IPv6 addresses, which need to be specified with
|
16
|
+
# a bounding []
|
15
17
|
SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
|
16
18
|
|
17
19
|
DEFAULT_PORT = 11_211
|
18
20
|
DEFAULT_WEIGHT = 1
|
19
21
|
|
20
22
|
def self.parse(str)
|
23
|
+
return parse_non_uri(str) unless str.start_with?(MEMCACHED_URI_PROTOCOL)
|
24
|
+
|
25
|
+
parse_uri(str)
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.parse_uri(str)
|
29
|
+
uri = URI.parse(str)
|
30
|
+
auth_details = {
|
31
|
+
username: uri.user,
|
32
|
+
password: uri.password
|
33
|
+
}
|
34
|
+
[uri.host, normalize_port(uri.port), :tcp, DEFAULT_WEIGHT, auth_details]
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.parse_non_uri(str)
|
21
38
|
res = deconstruct_string(str)
|
22
39
|
|
23
|
-
hostname =
|
40
|
+
hostname = normalize_host_from_match(str, res)
|
24
41
|
if hostname.start_with?('/')
|
25
42
|
socket_type = :unix
|
26
43
|
port, weight = attributes_for_unix_socket(res)
|
@@ -28,7 +45,7 @@ module Dalli
|
|
28
45
|
socket_type = :tcp
|
29
46
|
port, weight = attributes_for_tcp_socket(res)
|
30
47
|
end
|
31
|
-
[hostname, port, weight,
|
48
|
+
[hostname, port, socket_type, weight, {}]
|
32
49
|
end
|
33
50
|
|
34
51
|
def self.deconstruct_string(str)
|
@@ -49,7 +66,7 @@ module Dalli
|
|
49
66
|
[normalize_port(res[3]), normalize_weight(res[4])]
|
50
67
|
end
|
51
68
|
|
52
|
-
def self.
|
69
|
+
def self.normalize_host_from_match(str, res)
|
53
70
|
raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
|
54
71
|
|
55
72
|
res[2] || res[1]
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'forwardable'
|
4
|
+
|
5
|
+
module Dalli
|
6
|
+
module Protocol
|
7
|
+
##
|
8
|
+
# Dalli::Protocol::ValueMarshaller compartmentalizes the logic for marshalling
|
9
|
+
# and unmarshalling unstructured data (values) to Memcached. It also enforces
|
10
|
+
# limits on the maximum size of marshalled data.
|
11
|
+
##
|
12
|
+
class ValueMarshaller
|
13
|
+
extend Forwardable
|
14
|
+
|
15
|
+
DEFAULTS = {
|
16
|
+
# max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
|
17
|
+
value_max_bytes: 1024 * 1024
|
18
|
+
}.freeze
|
19
|
+
|
20
|
+
OPTIONS = DEFAULTS.keys.freeze
|
21
|
+
|
22
|
+
def_delegators :@value_serializer, :serializer
|
23
|
+
def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
|
24
|
+
|
25
|
+
def initialize(client_options)
|
26
|
+
@value_serializer = ValueSerializer.new(client_options)
|
27
|
+
@value_compressor = ValueCompressor.new(client_options)
|
28
|
+
|
29
|
+
@marshal_options =
|
30
|
+
DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
|
31
|
+
end
|
32
|
+
|
33
|
+
def store(key, value, options = nil)
|
34
|
+
bitflags = 0
|
35
|
+
value, bitflags = @value_serializer.store(value, options, bitflags)
|
36
|
+
value, bitflags = @value_compressor.store(value, options, bitflags)
|
37
|
+
|
38
|
+
error_if_over_max_value_bytes(key, value)
|
39
|
+
[value, bitflags]
|
40
|
+
end
|
41
|
+
|
42
|
+
def retrieve(value, flags)
|
43
|
+
value = @value_compressor.retrieve(value, flags)
|
44
|
+
@value_serializer.retrieve(value, flags)
|
45
|
+
end
|
46
|
+
|
47
|
+
def value_max_bytes
|
48
|
+
@marshal_options[:value_max_bytes]
|
49
|
+
end
|
50
|
+
|
51
|
+
def error_if_over_max_value_bytes(key, value)
|
52
|
+
return if value.bytesize <= value_max_bytes
|
53
|
+
|
54
|
+
message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
|
55
|
+
raise Dalli::ValueOverMaxSize, message
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dalli
|
4
|
+
module Protocol
|
5
|
+
##
|
6
|
+
# Dalli::Protocol::ValueSerializer compartmentalizes the logic for managing
|
7
|
+
# serialization and deserialization of stored values. It manages interpreting
|
8
|
+
# relevant options from both client and request, determining whether to
|
9
|
+
# serialize/deserialize on store/retrieve, and processes bitflags as necessary.
|
10
|
+
##
|
11
|
+
class ValueSerializer
|
12
|
+
DEFAULTS = {
|
13
|
+
serializer: Marshal
|
14
|
+
}.freeze
|
15
|
+
|
16
|
+
OPTIONS = DEFAULTS.keys.freeze
|
17
|
+
|
18
|
+
# https://www.hjp.at/zettel/m/memcached_flags.rxml
|
19
|
+
# Looks like most clients use bit 0 to indicate native language serialization
|
20
|
+
FLAG_SERIALIZED = 0x1
|
21
|
+
|
22
|
+
attr_accessor :serialization_options
|
23
|
+
|
24
|
+
def initialize(protocol_options)
|
25
|
+
@serialization_options =
|
26
|
+
DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
|
27
|
+
end
|
28
|
+
|
29
|
+
def store(value, req_options, bitflags)
|
30
|
+
do_serialize = !(req_options && req_options[:raw])
|
31
|
+
store_value = do_serialize ? serialize_value(value) : value.to_s
|
32
|
+
bitflags |= FLAG_SERIALIZED if do_serialize
|
33
|
+
[store_value, bitflags]
|
34
|
+
end
|
35
|
+
|
36
|
+
# TODO: Some of these error messages need to be validated. It's not obvious
|
37
|
+
# that all of them are actually generated by the invoked code
|
38
|
+
# in current systems
|
39
|
+
# rubocop:disable Layout/LineLength
|
40
|
+
TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
|
41
|
+
ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
|
42
|
+
NAME_ERR_STR = 'uninitialized constant'
|
43
|
+
# rubocop:enable Layout/LineLength
|
44
|
+
|
45
|
+
def retrieve(value, bitflags)
|
46
|
+
serialized = (bitflags & FLAG_SERIALIZED) != 0
|
47
|
+
serialized ? serializer.load(value) : value
|
48
|
+
rescue TypeError => e
|
49
|
+
filter_type_error(e)
|
50
|
+
rescue ArgumentError => e
|
51
|
+
filter_argument_error(e)
|
52
|
+
rescue NameError => e
|
53
|
+
filter_name_error(e)
|
54
|
+
end
|
55
|
+
|
56
|
+
def filter_type_error(err)
|
57
|
+
raise err unless TYPE_ERR_REGEXP.match?(err.message)
|
58
|
+
|
59
|
+
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
60
|
+
end
|
61
|
+
|
62
|
+
def filter_argument_error(err)
|
63
|
+
raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
|
64
|
+
|
65
|
+
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
66
|
+
end
|
67
|
+
|
68
|
+
def filter_name_error(err)
|
69
|
+
raise err unless err.message.include?(NAME_ERR_STR)
|
70
|
+
|
71
|
+
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
72
|
+
end
|
73
|
+
|
74
|
+
def serializer
|
75
|
+
@serialization_options[:serializer]
|
76
|
+
end
|
77
|
+
|
78
|
+
def serialize_value(value)
|
79
|
+
serializer.dump(value)
|
80
|
+
rescue Timeout::Error => e
|
81
|
+
raise e
|
82
|
+
rescue StandardError => e
|
83
|
+
# Serializing can throw several different types of generic Ruby exceptions.
|
84
|
+
# Convert to a specific exception so we can special case it higher up the stack.
|
85
|
+
exc = Dalli::MarshalError.new(e.message)
|
86
|
+
exc.set_backtrace e.backtrace
|
87
|
+
raise exc
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
data/lib/dalli/protocol.rb
CHANGED
@@ -2,8 +2,7 @@
|
|
2
2
|
|
3
3
|
module Dalli
|
4
4
|
module Protocol
|
5
|
-
#
|
6
|
-
|
7
|
-
NOT_FOUND = NilObject.new
|
5
|
+
# Preserved for backwards compatibility. Should be removed in 4.0
|
6
|
+
NOT_FOUND = ::Dalli::NOT_FOUND
|
8
7
|
end
|
9
8
|
end
|
data/lib/dalli/ring.rb
CHANGED
@@ -1,10 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require 'digest/sha1'
|
4
|
+
require 'zlib'
|
5
5
|
|
6
6
|
module Dalli
|
7
|
+
##
|
8
|
+
# An implementation of a consistent hash ring, designed to minimize
|
9
|
+
# the cache miss impact of adding or removing servers from the ring.
|
10
|
+
# That is, adding or removing a server from the ring should impact
|
11
|
+
# the key -> server mapping of ~ 1/N of the stored keys where N is the
|
12
|
+
# number of servers in the ring. This is done by creating a large
|
13
|
+
# number of "points" per server, distributed over the space
|
14
|
+
# 0x00000000 - 0xFFFFFFFF. For a given key, we calculate the CRC32
|
15
|
+
# hash, and find the nearest "point" that is less than or equal to the
|
16
|
+
# the key's hash. In this implemetation, each "point" is represented
|
17
|
+
# by a Dalli::Ring::Entry.
|
18
|
+
##
|
7
19
|
class Ring
|
20
|
+
# The number of entries on the continuum created per server
|
21
|
+
# in an equally weighted scenario.
|
8
22
|
POINTS_PER_SERVER = 160 # this is the default in libmemcached
|
9
23
|
|
10
24
|
attr_accessor :servers, :continuum
|
@@ -12,45 +26,48 @@ module Dalli
|
|
12
26
|
def initialize(servers, options)
|
13
27
|
@servers = servers
|
14
28
|
@continuum = nil
|
15
|
-
if servers.size > 1
|
16
|
-
total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
|
17
|
-
continuum = []
|
18
|
-
servers.each do |server|
|
19
|
-
entry_count_for(server, servers.size, total_weight).times do |idx|
|
20
|
-
hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
|
21
|
-
value = Integer("0x#{hash[0..7]}")
|
22
|
-
continuum << Dalli::Ring::Entry.new(value, server)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
@continuum = continuum.sort_by(&:value)
|
26
|
-
end
|
29
|
+
@continuum = build_continuum(servers) if servers.size > 1
|
27
30
|
|
28
31
|
threadsafe! unless options[:threadsafe] == false
|
29
32
|
@failover = options[:failover] != false
|
30
33
|
end
|
31
34
|
|
32
35
|
def server_for_key(key)
|
33
|
-
if @continuum
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
server =
|
50
|
-
|
36
|
+
server = if @continuum
|
37
|
+
server_from_continuum(key)
|
38
|
+
else
|
39
|
+
@servers.first
|
40
|
+
end
|
41
|
+
|
42
|
+
# Note that the call to alive? has the side effect of initializing
|
43
|
+
# the socket
|
44
|
+
return server if server&.alive?
|
45
|
+
|
46
|
+
raise Dalli::RingError, 'No server available'
|
47
|
+
end
|
48
|
+
|
49
|
+
def server_from_continuum(key)
|
50
|
+
hkey = hash_for(key)
|
51
|
+
20.times do |try|
|
52
|
+
server = server_for_hash_key(hkey)
|
53
|
+
|
54
|
+
# Note that the call to alive? has the side effect of initializing
|
55
|
+
# the socket
|
56
|
+
return server if server.alive?
|
57
|
+
break unless @failover
|
58
|
+
|
59
|
+
hkey = hash_for("#{try}#{key}")
|
51
60
|
end
|
61
|
+
nil
|
62
|
+
end
|
52
63
|
|
53
|
-
|
64
|
+
def keys_grouped_by_server(key_arr)
|
65
|
+
key_arr.group_by do |key|
|
66
|
+
server_for_key(key)
|
67
|
+
rescue Dalli::RingError
|
68
|
+
Dalli.logger.debug { "unable to get key #{key}" }
|
69
|
+
nil
|
70
|
+
end
|
54
71
|
end
|
55
72
|
|
56
73
|
def lock
|
@@ -62,6 +79,23 @@ module Dalli
|
|
62
79
|
end
|
63
80
|
end
|
64
81
|
|
82
|
+
def pipeline_consume_and_ignore_responses
|
83
|
+
@servers.each do |s|
|
84
|
+
s.request(:noop)
|
85
|
+
rescue Dalli::NetworkError
|
86
|
+
# Ignore this error, as it indicates the socket is unavailable
|
87
|
+
# and there's no need to flush
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def socket_timeout
|
92
|
+
@servers.first.socket_timeout
|
93
|
+
end
|
94
|
+
|
95
|
+
def close
|
96
|
+
@servers.each(&:close)
|
97
|
+
end
|
98
|
+
|
65
99
|
private
|
66
100
|
|
67
101
|
def threadsafe!
|
@@ -78,9 +112,35 @@ module Dalli
|
|
78
112
|
((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
|
79
113
|
end
|
80
114
|
|
115
|
+
def server_for_hash_key(hash_key)
|
116
|
+
# Find the closest index in the Ring with value <= the given value
|
117
|
+
entryidx = @continuum.bsearch_index { |entry| entry.value > hash_key }
|
118
|
+
if entryidx.nil?
|
119
|
+
entryidx = @continuum.size - 1
|
120
|
+
else
|
121
|
+
entryidx -= 1
|
122
|
+
end
|
123
|
+
@continuum[entryidx].server
|
124
|
+
end
|
125
|
+
|
126
|
+
def build_continuum(servers)
|
127
|
+
continuum = []
|
128
|
+
total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
|
129
|
+
servers.each do |server|
|
130
|
+
entry_count_for(server, servers.size, total_weight).times do |idx|
|
131
|
+
hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
|
132
|
+
value = Integer("0x#{hash[0..7]}")
|
133
|
+
continuum << Dalli::Ring::Entry.new(value, server)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
continuum.sort_by(&:value)
|
137
|
+
end
|
138
|
+
|
139
|
+
##
|
140
|
+
# Represents a point in the consistent hash ring implementation.
|
141
|
+
##
|
81
142
|
class Entry
|
82
|
-
attr_reader :value
|
83
|
-
attr_reader :server
|
143
|
+
attr_reader :value, :server
|
84
144
|
|
85
145
|
def initialize(val, srv)
|
86
146
|
@value = val
|
data/lib/dalli/server.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module Dalli
|
4
|
-
warn
|
3
|
+
module Dalli # rubocop:disable Style/Documentation
|
4
|
+
warn 'Dalli::Server is deprecated, use Dalli::Protocol::Binary instead'
|
5
5
|
Server = Protocol::Binary
|
6
6
|
end
|