dalli 2.7.11 → 3.0.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of dalli might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Gemfile +7 -6
- data/History.md +69 -0
- data/README.md +26 -200
- data/lib/dalli/cas/client.rb +1 -57
- data/lib/dalli/client.rb +272 -209
- data/lib/dalli/compressor.rb +12 -2
- data/lib/dalli/key_manager.rb +113 -0
- data/lib/dalli/options.rb +3 -4
- data/lib/dalli/protocol/binary/request_formatter.rb +109 -0
- data/lib/dalli/protocol/binary/response_processor.rb +149 -0
- data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
- data/lib/dalli/protocol/binary.rb +544 -0
- data/lib/dalli/protocol/server_config_parser.rb +84 -0
- data/lib/dalli/protocol/ttl_sanitizer.rb +45 -0
- data/lib/dalli/protocol/value_compressor.rb +85 -0
- data/lib/dalli/protocol/value_marshaller.rb +59 -0
- data/lib/dalli/protocol/value_serializer.rb +91 -0
- data/lib/dalli/protocol.rb +8 -0
- data/lib/dalli/ring.rb +86 -81
- data/lib/dalli/server.rb +3 -749
- data/lib/dalli/servers_arg_normalizer.rb +54 -0
- data/lib/dalli/socket.rb +115 -137
- data/lib/dalli/version.rb +4 -1
- data/lib/dalli.rb +32 -14
- data/lib/rack/session/dalli.rb +46 -55
- metadata +103 -10
- data/lib/action_dispatch/middleware/session/dalli_store.rb +0 -82
- data/lib/active_support/cache/dalli_store.rb +0 -441
- data/lib/dalli/railtie.rb +0 -8
data/lib/dalli/client.rb
CHANGED
@@ -1,17 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'digest/md5'
|
3
4
|
require 'set'
|
4
5
|
|
5
6
|
# encoding: ascii
|
6
7
|
module Dalli
|
8
|
+
##
|
9
|
+
# Dalli::Client is the main class which developers will use to interact with
|
10
|
+
# Memcached.
|
11
|
+
##
|
7
12
|
class Client
|
8
|
-
|
9
13
|
##
|
10
14
|
# Dalli::Client is the main class which developers will use to interact with
|
11
15
|
# the memcached server. Usage:
|
12
16
|
#
|
13
|
-
# Dalli::Client.new(['localhost:11211:10',
|
14
|
-
#
|
17
|
+
# Dalli::Client.new(['localhost:11211:10',
|
18
|
+
# 'cache-2.example.com:11211:5',
|
19
|
+
# '192.168.0.1:22122:5',
|
20
|
+
# '/var/run/memcached/socket'],
|
21
|
+
# failover: true, expires_in: 300)
|
15
22
|
#
|
16
23
|
# servers is an Array of "host:port:weight" where weight allows you to distribute cache unevenly.
|
17
24
|
# Both weight and port are optional. If you pass in nil, Dalli will use the <tt>MEMCACHE_SERVERS</tt>
|
@@ -24,16 +31,25 @@ module Dalli
|
|
24
31
|
# - :namespace - prepend each key with this value to provide simple namespacing.
|
25
32
|
# - :failover - if a server is down, look for and store values on another server in the ring. Default: true.
|
26
33
|
# - :threadsafe - ensure that only one thread is actively using a socket at a time. Default: true.
|
27
|
-
# - :expires_in - default TTL in seconds if you do not pass TTL as a parameter to an individual operation, defaults
|
28
|
-
#
|
34
|
+
# - :expires_in - default TTL in seconds if you do not pass TTL as a parameter to an individual operation, defaults
|
35
|
+
# to 0 or forever.
|
36
|
+
# - :compress - if true Dalli will compress values larger than compression_min_size bytes before sending them
|
37
|
+
# to memcached. Default: true.
|
38
|
+
# - :compression_min_size - the minimum size (in bytes) for which Dalli will compress values sent to Memcached.
|
39
|
+
# Defaults to 4K.
|
29
40
|
# - :serializer - defaults to Marshal
|
30
|
-
# - :compressor - defaults to
|
31
|
-
# - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for
|
32
|
-
#
|
41
|
+
# - :compressor - defaults to Dalli::Compressor, a Zlib-based implementation
|
42
|
+
# - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for
|
43
|
+
# #fetch operations.
|
44
|
+
# - :digest_class - defaults to Digest::MD5, allows you to pass in an object that responds to the hexdigest method,
|
45
|
+
# useful for injecting a FIPS compliant hash object.
|
46
|
+
# - :protocol_implementation - defaults to Dalli::Protocol::Binary which uses the binary protocol. Allows you to
|
47
|
+
# pass an alternative implementation using another protocol.
|
33
48
|
#
|
34
|
-
def initialize(servers=nil, options={})
|
35
|
-
@servers = normalize_servers(servers
|
49
|
+
def initialize(servers = nil, options = {})
|
50
|
+
@servers = ::Dalli::ServersArgNormalizer.normalize_servers(servers)
|
36
51
|
@options = normalize_options(options)
|
52
|
+
@key_manager = ::Dalli::KeyManager.new(options)
|
37
53
|
@ring = nil
|
38
54
|
end
|
39
55
|
|
@@ -47,16 +63,18 @@ module Dalli
|
|
47
63
|
# pipelined as Dalli will use 'quiet' operations where possible.
|
48
64
|
# Currently supports the set, add, replace and delete operations.
|
49
65
|
def multi
|
50
|
-
old
|
66
|
+
old = Thread.current[:dalli_multi]
|
67
|
+
Thread.current[:dalli_multi] = true
|
51
68
|
yield
|
52
69
|
ensure
|
70
|
+
@ring&.flush_multi_responses
|
53
71
|
Thread.current[:dalli_multi] = old
|
54
72
|
end
|
55
73
|
|
56
74
|
##
|
57
75
|
# Get the value associated with the key.
|
58
76
|
# If a value is not found, then +nil+ is returned.
|
59
|
-
def get(key, options=nil)
|
77
|
+
def get(key, options = nil)
|
60
78
|
perform(:get, key, options)
|
61
79
|
end
|
62
80
|
|
@@ -65,20 +83,21 @@ module Dalli
|
|
65
83
|
# If a block is given, yields key/value pairs one at a time.
|
66
84
|
# Otherwise returns a hash of { 'key' => 'value', 'key2' => 'value1' }
|
67
85
|
def get_multi(*keys)
|
68
|
-
|
69
|
-
|
86
|
+
keys.flatten!
|
87
|
+
keys.compact!
|
88
|
+
|
89
|
+
return {} if keys.empty?
|
70
90
|
|
71
|
-
return {} if check_keys.empty?
|
72
91
|
if block_given?
|
73
|
-
get_multi_yielder(keys) {|k, data| yield k, data.first}
|
92
|
+
get_multi_yielder(keys) { |k, data| yield k, data.first }
|
74
93
|
else
|
75
|
-
|
76
|
-
get_multi_yielder(keys) {|k, data| hash[k] = data.first}
|
94
|
+
{}.tap do |hash|
|
95
|
+
get_multi_yielder(keys) { |k, data| hash[k] = data.first }
|
77
96
|
end
|
78
97
|
end
|
79
98
|
end
|
80
99
|
|
81
|
-
CACHE_NILS = {cache_nils: true}.freeze
|
100
|
+
CACHE_NILS = { cache_nils: true }.freeze
|
82
101
|
|
83
102
|
# Fetch the value associated with the key.
|
84
103
|
# If a value is found, then it is returned.
|
@@ -88,19 +107,24 @@ module Dalli
|
|
88
107
|
# If a value is not found (or if the found value is nil and :cache_nils is false)
|
89
108
|
# and a block is given, the block will be invoked and its return value
|
90
109
|
# written to the cache and returned.
|
91
|
-
def fetch(key, ttl=nil,
|
92
|
-
|
93
|
-
val = get(key,
|
94
|
-
not_found
|
95
|
-
val == Dalli::Server::NOT_FOUND :
|
96
|
-
val.nil?
|
97
|
-
if not_found && block_given?
|
110
|
+
def fetch(key, ttl = nil, req_options = nil)
|
111
|
+
req_options = req_options.nil? ? CACHE_NILS : req_options.merge(CACHE_NILS) if cache_nils
|
112
|
+
val = get(key, req_options)
|
113
|
+
if not_found?(val) && block_given?
|
98
114
|
val = yield
|
99
|
-
add(key, val, ttl_or_default(ttl),
|
115
|
+
add(key, val, ttl_or_default(ttl), req_options)
|
100
116
|
end
|
101
117
|
val
|
102
118
|
end
|
103
119
|
|
120
|
+
def not_found?(val)
|
121
|
+
cache_nils ? val == ::Dalli::NOT_FOUND : val.nil?
|
122
|
+
end
|
123
|
+
|
124
|
+
def cache_nils
|
125
|
+
@options[:cache_nils]
|
126
|
+
end
|
127
|
+
|
104
128
|
##
|
105
129
|
# compare and swap values using optimistic locking.
|
106
130
|
# Fetch the existing value for key.
|
@@ -112,7 +136,7 @@ module Dalli
|
|
112
136
|
# - nil if the key did not exist.
|
113
137
|
# - false if the value was changed by someone else.
|
114
138
|
# - true if the value was successfully updated.
|
115
|
-
def cas(key, ttl=nil, options=nil, &block)
|
139
|
+
def cas(key, ttl = nil, options = nil, &block)
|
116
140
|
cas_core(key, false, ttl, options, &block)
|
117
141
|
end
|
118
142
|
|
@@ -123,25 +147,25 @@ module Dalli
|
|
123
147
|
# Returns:
|
124
148
|
# - false if the value was changed by someone else.
|
125
149
|
# - true if the value was successfully updated.
|
126
|
-
def cas!(key, ttl=nil, options=nil, &block)
|
150
|
+
def cas!(key, ttl = nil, options = nil, &block)
|
127
151
|
cas_core(key, true, ttl, options, &block)
|
128
152
|
end
|
129
153
|
|
130
|
-
def set(key, value, ttl=nil, options=nil)
|
154
|
+
def set(key, value, ttl = nil, options = nil)
|
131
155
|
perform(:set, key, value, ttl_or_default(ttl), 0, options)
|
132
156
|
end
|
133
157
|
|
134
158
|
##
|
135
159
|
# Conditionally add a key/value pair, if the key does not already exist
|
136
160
|
# on the server. Returns truthy if the operation succeeded.
|
137
|
-
def add(key, value, ttl=nil, options=nil)
|
161
|
+
def add(key, value, ttl = nil, options = nil)
|
138
162
|
perform(:add, key, value, ttl_or_default(ttl), options)
|
139
163
|
end
|
140
164
|
|
141
165
|
##
|
142
166
|
# Conditionally add a key/value pair, only if the key already exists
|
143
167
|
# on the server. Returns truthy if the operation succeeded.
|
144
|
-
def replace(key, value, ttl=nil, options=nil)
|
168
|
+
def replace(key, value, ttl = nil, options = nil)
|
145
169
|
perform(:replace, key, value, ttl_or_default(ttl), 0, options)
|
146
170
|
end
|
147
171
|
|
@@ -163,12 +187,12 @@ module Dalli
|
|
163
187
|
perform(:prepend, key, value.to_s)
|
164
188
|
end
|
165
189
|
|
166
|
-
def flush(delay=0)
|
190
|
+
def flush(delay = 0)
|
167
191
|
time = -delay
|
168
192
|
ring.servers.map { |s| s.request(:flush, time += delay) }
|
169
193
|
end
|
170
194
|
|
171
|
-
|
195
|
+
alias flush_all flush
|
172
196
|
|
173
197
|
##
|
174
198
|
# Incr adds the given amount to the counter on the memcached server.
|
@@ -181,8 +205,9 @@ module Dalli
|
|
181
205
|
# Note that the ttl will only apply if the counter does not already
|
182
206
|
# exist. To increase an existing counter and update its TTL, use
|
183
207
|
# #cas.
|
184
|
-
def incr(key, amt=1, ttl=nil, default=nil)
|
185
|
-
raise ArgumentError, "Positive values only: #{amt}" if amt
|
208
|
+
def incr(key, amt = 1, ttl = nil, default = nil)
|
209
|
+
raise ArgumentError, "Positive values only: #{amt}" if amt.negative?
|
210
|
+
|
186
211
|
perform(:incr, key, amt.to_i, ttl_or_default(ttl), default)
|
187
212
|
end
|
188
213
|
|
@@ -200,8 +225,9 @@ module Dalli
|
|
200
225
|
# Note that the ttl will only apply if the counter does not already
|
201
226
|
# exist. To decrease an existing counter and update its TTL, use
|
202
227
|
# #cas.
|
203
|
-
def decr(key, amt=1, ttl=nil, default=nil)
|
204
|
-
raise ArgumentError, "Positive values only: #{amt}" if amt
|
228
|
+
def decr(key, amt = 1, ttl = nil, default = nil)
|
229
|
+
raise ArgumentError, "Positive values only: #{amt}" if amt.negative?
|
230
|
+
|
205
231
|
perform(:decr, key, amt.to_i, ttl_or_default(ttl), default)
|
206
232
|
end
|
207
233
|
|
@@ -209,20 +235,28 @@ module Dalli
|
|
209
235
|
# Touch updates expiration time for a given key.
|
210
236
|
#
|
211
237
|
# Returns true if key exists, otherwise nil.
|
212
|
-
def touch(key, ttl=nil)
|
238
|
+
def touch(key, ttl = nil)
|
213
239
|
resp = perform(:touch, key, ttl_or_default(ttl))
|
214
240
|
resp.nil? ? nil : true
|
215
241
|
end
|
216
242
|
|
243
|
+
##
|
244
|
+
# Gat (get and touch) fetch an item and simultaneously update its expiration time.
|
245
|
+
#
|
246
|
+
# If a value is not found, then +nil+ is returned.
|
247
|
+
def gat(key, ttl = nil)
|
248
|
+
perform(:gat, key, ttl_or_default(ttl))
|
249
|
+
end
|
250
|
+
|
217
251
|
##
|
218
252
|
# Collect the stats for each server.
|
219
253
|
# You can optionally pass a type including :items, :slabs or :settings to get specific stats
|
220
254
|
# Returns a hash like { 'hostname:port' => { 'stat1' => 'value1', ... }, 'hostname2:port' => { ... } }
|
221
|
-
def stats(type=nil)
|
222
|
-
type = nil
|
255
|
+
def stats(type = nil)
|
256
|
+
type = nil unless [nil, :items, :slabs, :settings].include? type
|
223
257
|
values = {}
|
224
258
|
ring.servers.each do |server|
|
225
|
-
values[
|
259
|
+
values[server.name.to_s] = server.alive? ? server.request(:stats, type.to_s) : nil
|
226
260
|
end
|
227
261
|
values
|
228
262
|
end
|
@@ -238,7 +272,7 @@ module Dalli
|
|
238
272
|
##
|
239
273
|
## Make sure memcache servers are alive, or raise an Dalli::RingError
|
240
274
|
def alive!
|
241
|
-
ring.server_for_key(
|
275
|
+
ring.server_for_key('')
|
242
276
|
end
|
243
277
|
|
244
278
|
##
|
@@ -246,21 +280,73 @@ module Dalli
|
|
246
280
|
def version
|
247
281
|
values = {}
|
248
282
|
ring.servers.each do |server|
|
249
|
-
values[
|
283
|
+
values[server.name.to_s] = server.alive? ? server.request(:version) : nil
|
250
284
|
end
|
251
285
|
values
|
252
286
|
end
|
253
287
|
|
288
|
+
##
|
289
|
+
# Get the value and CAS ID associated with the key. If a block is provided,
|
290
|
+
# value and CAS will be passed to the block.
|
291
|
+
def get_cas(key)
|
292
|
+
(value, cas) = perform(:cas, key)
|
293
|
+
value = nil if !value || value == 'Not found'
|
294
|
+
if block_given?
|
295
|
+
yield value, cas
|
296
|
+
else
|
297
|
+
[value, cas]
|
298
|
+
end
|
299
|
+
end
|
300
|
+
|
301
|
+
##
|
302
|
+
# Fetch multiple keys efficiently, including available metadata such as CAS.
|
303
|
+
# If a block is given, yields key/data pairs one a time. Data is an array:
|
304
|
+
# [value, cas_id]
|
305
|
+
# If no block is given, returns a hash of
|
306
|
+
# { 'key' => [value, cas_id] }
|
307
|
+
def get_multi_cas(*keys)
|
308
|
+
if block_given?
|
309
|
+
get_multi_yielder(keys) { |*args| yield(*args) }
|
310
|
+
else
|
311
|
+
{}.tap do |hash|
|
312
|
+
get_multi_yielder(keys) { |k, data| hash[k] = data }
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
##
|
318
|
+
# Set the key-value pair, verifying existing CAS.
|
319
|
+
# Returns the resulting CAS value if succeeded, and falsy otherwise.
|
320
|
+
def set_cas(key, value, cas, ttl = nil, options = nil)
|
321
|
+
ttl ||= @options[:expires_in].to_i
|
322
|
+
perform(:set, key, value, ttl, cas, options)
|
323
|
+
end
|
324
|
+
|
325
|
+
##
|
326
|
+
# Conditionally add a key/value pair, verifying existing CAS, only if the
|
327
|
+
# key already exists on the server. Returns the new CAS value if the
|
328
|
+
# operation succeeded, or falsy otherwise.
|
329
|
+
def replace_cas(key, value, cas, ttl = nil, options = nil)
|
330
|
+
ttl ||= @options[:expires_in].to_i
|
331
|
+
perform(:replace, key, value, ttl, cas, options)
|
332
|
+
end
|
333
|
+
|
334
|
+
# Delete a key/value pair, verifying existing CAS.
|
335
|
+
# Returns true if succeeded, and falsy otherwise.
|
336
|
+
def delete_cas(key, cas = 0)
|
337
|
+
perform(:delete, key, cas)
|
338
|
+
end
|
339
|
+
|
254
340
|
##
|
255
341
|
# Close our connection to each server.
|
256
342
|
# If you perform another operation after this, the connections will be re-established.
|
257
343
|
def close
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
344
|
+
return unless @ring
|
345
|
+
|
346
|
+
@ring.servers.each(&:close)
|
347
|
+
@ring = nil
|
262
348
|
end
|
263
|
-
|
349
|
+
alias reset close
|
264
350
|
|
265
351
|
# Stub method so a bare Dalli client can pretend to be a connection pool.
|
266
352
|
def with
|
@@ -269,10 +355,11 @@ module Dalli
|
|
269
355
|
|
270
356
|
private
|
271
357
|
|
272
|
-
def cas_core(key, always_set, ttl=nil, options=nil)
|
358
|
+
def cas_core(key, always_set, ttl = nil, options = nil)
|
273
359
|
(value, cas) = perform(:cas, key)
|
274
|
-
value =
|
360
|
+
value = nil if !value || value == 'Not found'
|
275
361
|
return if value.nil? && !always_set
|
362
|
+
|
276
363
|
newvalue = yield(value)
|
277
364
|
perform(:set, key, newvalue, ttl_or_default(ttl), cas, options)
|
278
365
|
end
|
@@ -283,201 +370,177 @@ module Dalli
|
|
283
370
|
raise ArgumentError, "Cannot convert ttl (#{ttl}) to an integer"
|
284
371
|
end
|
285
372
|
|
286
|
-
def
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
373
|
+
def ring
|
374
|
+
# TODO: This server initialization should probably be pushed down
|
375
|
+
# to the Ring
|
376
|
+
@ring ||= Dalli::Ring.new(
|
377
|
+
@servers.map do |s|
|
378
|
+
protocol_implementation.new(s, @options)
|
379
|
+
end, @options
|
380
|
+
)
|
381
|
+
end
|
382
|
+
|
383
|
+
def protocol_implementation
|
384
|
+
@protocol_implementation ||= @options.fetch(:protocol_implementation, Dalli::Protocol::Binary)
|
385
|
+
end
|
386
|
+
|
387
|
+
# Chokepoint method for instrumentation
|
388
|
+
def perform(*all_args)
|
389
|
+
return yield if block_given?
|
390
|
+
|
391
|
+
op, key, *args = all_args
|
392
|
+
|
393
|
+
key = key.to_s
|
394
|
+
key = @key_manager.validate_key(key)
|
395
|
+
|
396
|
+
server = ring.server_for_key(key)
|
397
|
+
server.request(op, key, *args)
|
398
|
+
rescue NetworkError => e
|
399
|
+
Dalli.logger.debug { e.inspect }
|
400
|
+
Dalli.logger.debug { 'retrying request with new server' }
|
401
|
+
retry
|
402
|
+
end
|
403
|
+
|
404
|
+
def normalize_options(opts)
|
405
|
+
begin
|
406
|
+
opts[:expires_in] = opts[:expires_in].to_i if opts[:expires_in]
|
407
|
+
rescue NoMethodError
|
408
|
+
raise ArgumentError, "cannot convert :expires_in => #{opts[:expires_in].inspect} to an integer"
|
294
409
|
end
|
295
|
-
|
410
|
+
opts
|
296
411
|
end
|
297
412
|
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
413
|
+
# TODO: Look at extracting below into separate MultiYielder class
|
414
|
+
|
415
|
+
##
|
416
|
+
# Yields, one at a time, keys and their values+attributes.
|
417
|
+
#
|
418
|
+
def get_multi_yielder(keys, &block)
|
419
|
+
return {} if keys.empty?
|
420
|
+
|
421
|
+
ring.lock do
|
422
|
+
groups = groups_for_keys(keys)
|
423
|
+
if (unfound_keys = groups.delete(nil))
|
424
|
+
Dalli.logger.debug do
|
425
|
+
"unable to get keys for #{unfound_keys.length} keys "\
|
426
|
+
'because no matching server was found'
|
427
|
+
end
|
428
|
+
end
|
429
|
+
make_multi_get_requests(groups)
|
430
|
+
|
431
|
+
servers = groups.keys
|
432
|
+
return if servers.empty?
|
433
|
+
|
434
|
+
# TODO: How does this exit on a NetworkError
|
435
|
+
servers = perform_multi_response_start(servers)
|
436
|
+
|
437
|
+
timeout = servers.first.options[:socket_timeout]
|
438
|
+
start_time = Time.now
|
439
|
+
loop do
|
440
|
+
# remove any dead servers
|
441
|
+
# TODO: Is this well behaved in a multi-threaded environment?
|
442
|
+
# Accessing the server socket like this seems problematic
|
443
|
+
servers.delete_if { |s| s.sock.nil? }
|
444
|
+
break if servers.empty?
|
445
|
+
|
446
|
+
servers = multi_yielder_loop(servers, start_time, timeout, &block)
|
447
|
+
end
|
448
|
+
end
|
449
|
+
rescue NetworkError => e
|
450
|
+
Dalli.logger.debug { e.inspect }
|
451
|
+
Dalli.logger.debug { 'retrying multi yielder because of timeout' }
|
452
|
+
retry
|
302
453
|
end
|
303
454
|
|
304
455
|
def make_multi_get_requests(groups)
|
305
456
|
groups.each do |server, keys_for_server|
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
# into perform to be meaningful
|
311
|
-
server.request(:send_multiget, keys_for_server)
|
312
|
-
rescue DalliError, NetworkError => e
|
313
|
-
Dalli.logger.debug { e.inspect }
|
314
|
-
Dalli.logger.debug { "unable to get keys for server #{server.name}" }
|
315
|
-
end
|
457
|
+
server.request(:send_multiget, keys_for_server)
|
458
|
+
rescue DalliError, NetworkError => e
|
459
|
+
Dalli.logger.debug { e.inspect }
|
460
|
+
Dalli.logger.debug { "unable to get keys for server #{server.name}" }
|
316
461
|
end
|
317
462
|
end
|
318
463
|
|
464
|
+
# raises Dalli::NetworkError
|
319
465
|
def perform_multi_response_start(servers)
|
466
|
+
deleted = []
|
467
|
+
|
320
468
|
servers.each do |server|
|
321
469
|
next unless server.alive?
|
470
|
+
|
322
471
|
begin
|
323
472
|
server.multi_response_start
|
324
|
-
rescue
|
473
|
+
rescue Dalli::NetworkError
|
474
|
+
abort_multi_response(servers)
|
475
|
+
raise
|
476
|
+
rescue Dalli::DalliError => e
|
325
477
|
Dalli.logger.debug { e.inspect }
|
326
|
-
Dalli.logger.debug {
|
327
|
-
|
478
|
+
Dalli.logger.debug { 'results from this server will be missing' }
|
479
|
+
deleted.append(server)
|
328
480
|
end
|
329
481
|
end
|
330
|
-
servers
|
331
|
-
end
|
332
482
|
|
333
|
-
|
334
|
-
# Normalizes the argument into an array of servers.
|
335
|
-
# If the argument is a string, or an array containing strings, it's expected that the URIs are comma separated e.g.
|
336
|
-
# "memcache1.example.com:11211,memcache2.example.com:11211,memcache3.example.com:11211"
|
337
|
-
def normalize_servers(servers)
|
338
|
-
Array(servers).flat_map do |server|
|
339
|
-
if server.is_a? String
|
340
|
-
server.split(",")
|
341
|
-
else
|
342
|
-
server
|
343
|
-
end
|
344
|
-
end
|
483
|
+
servers.delete_if { |server| deleted.include?(server) }
|
345
484
|
end
|
346
485
|
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
server_options = {}
|
351
|
-
if s =~ %r{\Amemcached://}
|
352
|
-
uri = URI.parse(s)
|
353
|
-
server_options[:username] = uri.user
|
354
|
-
server_options[:password] = uri.password
|
355
|
-
s = "#{uri.host}:#{uri.port}"
|
356
|
-
end
|
357
|
-
Dalli::Server.new(s, @options.merge(server_options))
|
358
|
-
end, @options
|
359
|
-
)
|
486
|
+
# Swallows Dalli::NetworkError
|
487
|
+
def abort_multi_response(servers)
|
488
|
+
servers.each(&:multi_response_abort)
|
360
489
|
end
|
361
490
|
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
key = validate_key(key)
|
369
|
-
begin
|
370
|
-
server = ring.server_for_key(key)
|
371
|
-
ret = server.request(op, key, *args)
|
372
|
-
ret
|
373
|
-
rescue NetworkError => e
|
374
|
-
Dalli.logger.debug { e.inspect }
|
375
|
-
Dalli.logger.debug { "retrying request with new server" }
|
376
|
-
retry
|
491
|
+
def multi_yielder_loop(servers, start_time, timeout, &block)
|
492
|
+
time_left = remaining_time(start_time, timeout)
|
493
|
+
readable_servers = servers_with_data(servers, time_left)
|
494
|
+
if readable_servers.empty?
|
495
|
+
abort_multi_connections_w_timeout(servers)
|
496
|
+
return readable_servers
|
377
497
|
end
|
378
|
-
end
|
379
498
|
|
380
|
-
|
381
|
-
|
382
|
-
key = key_with_namespace(key)
|
383
|
-
if key.length > 250
|
384
|
-
digest_class = @options[:digest_class] || ::Digest::MD5
|
385
|
-
max_length_before_namespace = 212 - (namespace || '').size
|
386
|
-
key = "#{key[0, max_length_before_namespace]}:md5:#{digest_class.hexdigest(key)}"
|
499
|
+
readable_servers.each do |server|
|
500
|
+
servers.delete(server) if respond_to_readable_server(server, &block)
|
387
501
|
end
|
388
|
-
|
502
|
+
servers
|
503
|
+
rescue NetworkError
|
504
|
+
abort_multi_response(servers)
|
505
|
+
raise
|
389
506
|
end
|
390
507
|
|
391
|
-
def
|
392
|
-
|
393
|
-
|
508
|
+
def remaining_time(start, timeout)
|
509
|
+
elapsed = Time.now - start
|
510
|
+
return 0 if elapsed > timeout
|
394
511
|
|
395
|
-
|
396
|
-
(ns = namespace) ? key.sub(%r(\A#{Regexp.escape ns}:), '') : key
|
512
|
+
timeout - elapsed
|
397
513
|
end
|
398
514
|
|
399
|
-
|
400
|
-
|
401
|
-
|
515
|
+
# Swallows Dalli::NetworkError
|
516
|
+
def abort_multi_connections_w_timeout(servers)
|
517
|
+
abort_multi_response(servers)
|
518
|
+
servers.each do |server|
|
519
|
+
Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
|
520
|
+
end
|
521
|
+
|
522
|
+
true # Required to simplify caller
|
402
523
|
end
|
403
524
|
|
404
|
-
def
|
405
|
-
|
406
|
-
|
407
|
-
opts[:compress] = opts.delete(:compression)
|
525
|
+
def respond_to_readable_server(server)
|
526
|
+
server.multi_response_nonblock.each_pair do |key, value_list|
|
527
|
+
yield @key_manager.key_without_namespace(key), value_list
|
408
528
|
end
|
409
|
-
|
410
|
-
|
411
|
-
rescue NoMethodError
|
412
|
-
raise ArgumentError, "cannot convert :expires_in => #{opts[:expires_in].inspect} to an integer"
|
413
|
-
end
|
414
|
-
if opts[:digest_class] && !opts[:digest_class].respond_to?(:hexdigest)
|
415
|
-
raise ArgumentError, "The digest_class object must respond to the hexdigest method"
|
416
|
-
end
|
417
|
-
opts
|
529
|
+
|
530
|
+
server.multi_response_completed?
|
418
531
|
end
|
419
532
|
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
ring.lock do
|
426
|
-
begin
|
427
|
-
groups = groups_for_keys(keys)
|
428
|
-
if unfound_keys = groups.delete(nil)
|
429
|
-
Dalli.logger.debug { "unable to get keys for #{unfound_keys.length} keys because no matching server was found" }
|
430
|
-
end
|
431
|
-
make_multi_get_requests(groups)
|
432
|
-
|
433
|
-
servers = groups.keys
|
434
|
-
return if servers.empty?
|
435
|
-
servers = perform_multi_response_start(servers)
|
436
|
-
|
437
|
-
start = Time.now
|
438
|
-
while true
|
439
|
-
# remove any dead servers
|
440
|
-
servers.delete_if { |s| s.sock.nil? }
|
441
|
-
break if servers.empty?
|
442
|
-
|
443
|
-
# calculate remaining timeout
|
444
|
-
elapsed = Time.now - start
|
445
|
-
timeout = servers.first.options[:socket_timeout]
|
446
|
-
time_left = (elapsed > timeout) ? 0 : timeout - elapsed
|
447
|
-
|
448
|
-
sockets = servers.map(&:sock)
|
449
|
-
readable, _ = IO.select(sockets, nil, nil, time_left)
|
450
|
-
|
451
|
-
if readable.nil?
|
452
|
-
# no response within timeout; abort pending connections
|
453
|
-
servers.each do |server|
|
454
|
-
Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
|
455
|
-
server.multi_response_abort
|
456
|
-
end
|
457
|
-
break
|
458
|
-
|
459
|
-
else
|
460
|
-
readable.each do |sock|
|
461
|
-
server = sock.server
|
462
|
-
|
463
|
-
begin
|
464
|
-
server.multi_response_nonblock.each_pair do |key, value_list|
|
465
|
-
yield key_without_namespace(key), value_list
|
466
|
-
end
|
467
|
-
|
468
|
-
if server.multi_response_completed?
|
469
|
-
servers.delete(server)
|
470
|
-
end
|
471
|
-
rescue NetworkError
|
472
|
-
servers.delete(server)
|
473
|
-
end
|
474
|
-
end
|
475
|
-
end
|
476
|
-
end
|
477
|
-
end
|
478
|
-
end
|
479
|
-
end
|
533
|
+
def servers_with_data(servers, timeout)
|
534
|
+
readable, = IO.select(servers.map(&:sock), nil, nil, timeout)
|
535
|
+
return [] if readable.nil?
|
536
|
+
|
537
|
+
readable.map(&:server)
|
480
538
|
end
|
481
539
|
|
540
|
+
def groups_for_keys(*keys)
|
541
|
+
keys.flatten!
|
542
|
+
keys.map! { |a| @key_manager.validate_key(a.to_s) }
|
543
|
+
ring.keys_grouped_by_server(keys)
|
544
|
+
end
|
482
545
|
end
|
483
546
|
end
|