dalli 2.7.6 → 3.2.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of dalli might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Gemfile +5 -1
- data/History.md +175 -0
- data/README.md +27 -213
- data/lib/dalli/cas/client.rb +2 -57
- data/lib/dalli/client.rb +228 -254
- data/lib/dalli/compressor.rb +13 -2
- data/lib/dalli/key_manager.rb +113 -0
- data/lib/dalli/options.rb +7 -7
- data/lib/dalli/pipelined_getter.rb +177 -0
- data/lib/dalli/protocol/base.rb +241 -0
- data/lib/dalli/protocol/binary/request_formatter.rb +117 -0
- data/lib/dalli/protocol/binary/response_header.rb +36 -0
- data/lib/dalli/protocol/binary/response_processor.rb +239 -0
- data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
- data/lib/dalli/protocol/binary.rb +173 -0
- data/lib/dalli/protocol/connection_manager.rb +252 -0
- data/lib/dalli/protocol/meta/key_regularizer.rb +31 -0
- data/lib/dalli/protocol/meta/request_formatter.rb +108 -0
- data/lib/dalli/protocol/meta/response_processor.rb +211 -0
- data/lib/dalli/protocol/meta.rb +177 -0
- data/lib/dalli/protocol/response_buffer.rb +54 -0
- data/lib/dalli/protocol/server_config_parser.rb +84 -0
- data/lib/dalli/protocol/ttl_sanitizer.rb +45 -0
- data/lib/dalli/protocol/value_compressor.rb +85 -0
- data/lib/dalli/protocol/value_marshaller.rb +59 -0
- data/lib/dalli/protocol/value_serializer.rb +91 -0
- data/lib/dalli/protocol.rb +8 -0
- data/lib/dalli/ring.rb +95 -84
- data/lib/dalli/server.rb +4 -743
- data/lib/dalli/servers_arg_normalizer.rb +54 -0
- data/lib/dalli/socket.rb +118 -130
- data/lib/dalli/version.rb +5 -1
- data/lib/dalli.rb +45 -14
- data/lib/rack/session/dalli.rb +156 -50
- metadata +64 -27
- data/lib/action_dispatch/middleware/session/dalli_store.rb +0 -81
- data/lib/active_support/cache/dalli_store.rb +0 -403
- data/lib/dalli/railtie.rb +0 -7
data/lib/dalli/client.rb
CHANGED
@@ -1,16 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'digest/md5'
|
2
4
|
require 'set'
|
3
5
|
|
4
6
|
# encoding: ascii
|
5
7
|
module Dalli
|
8
|
+
##
|
9
|
+
# Dalli::Client is the main class which developers will use to interact with
|
10
|
+
# Memcached.
|
11
|
+
##
|
6
12
|
class Client
|
7
|
-
|
8
13
|
##
|
9
14
|
# Dalli::Client is the main class which developers will use to interact with
|
10
15
|
# the memcached server. Usage:
|
11
16
|
#
|
12
|
-
# Dalli::Client.new(['localhost:11211:10',
|
13
|
-
#
|
17
|
+
# Dalli::Client.new(['localhost:11211:10',
|
18
|
+
# 'cache-2.example.com:11211:5',
|
19
|
+
# '192.168.0.1:22122:5',
|
20
|
+
# '/var/run/memcached/socket'],
|
21
|
+
# failover: true, expires_in: 300)
|
14
22
|
#
|
15
23
|
# servers is an Array of "host:port:weight" where weight allows you to distribute cache unevenly.
|
16
24
|
# Both weight and port are optional. If you pass in nil, Dalli will use the <tt>MEMCACHE_SERVERS</tt>
|
@@ -23,15 +31,25 @@ module Dalli
|
|
23
31
|
# - :namespace - prepend each key with this value to provide simple namespacing.
|
24
32
|
# - :failover - if a server is down, look for and store values on another server in the ring. Default: true.
|
25
33
|
# - :threadsafe - ensure that only one thread is actively using a socket at a time. Default: true.
|
26
|
-
# - :expires_in - default TTL in seconds if you do not pass TTL as a parameter to an individual operation, defaults
|
27
|
-
#
|
34
|
+
# - :expires_in - default TTL in seconds if you do not pass TTL as a parameter to an individual operation, defaults
|
35
|
+
# to 0 or forever.
|
36
|
+
# - :compress - if true Dalli will compress values larger than compression_min_size bytes before sending them
|
37
|
+
# to memcached. Default: true.
|
38
|
+
# - :compression_min_size - the minimum size (in bytes) for which Dalli will compress values sent to Memcached.
|
39
|
+
# Defaults to 4K.
|
28
40
|
# - :serializer - defaults to Marshal
|
29
|
-
# - :compressor - defaults to
|
30
|
-
# - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for
|
41
|
+
# - :compressor - defaults to Dalli::Compressor, a Zlib-based implementation
|
42
|
+
# - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for
|
43
|
+
# #fetch operations.
|
44
|
+
# - :digest_class - defaults to Digest::MD5, allows you to pass in an object that responds to the hexdigest method,
|
45
|
+
# useful for injecting a FIPS compliant hash object.
|
46
|
+
# - :protocol - one of either :binary or :meta, defaulting to :binary. This sets the protocol that Dalli uses
|
47
|
+
# to communicate with memcached.
|
31
48
|
#
|
32
|
-
def initialize(servers=nil, options={})
|
33
|
-
@
|
49
|
+
def initialize(servers = nil, options = {})
|
50
|
+
@normalized_servers = ::Dalli::ServersArgNormalizer.normalize_servers(servers)
|
34
51
|
@options = normalize_options(options)
|
52
|
+
@key_manager = ::Dalli::KeyManager.new(@options)
|
35
53
|
@ring = nil
|
36
54
|
end
|
37
55
|
|
@@ -40,22 +58,37 @@ module Dalli
|
|
40
58
|
#
|
41
59
|
|
42
60
|
##
|
43
|
-
#
|
44
|
-
#
|
45
|
-
|
46
|
-
|
47
|
-
def multi
|
48
|
-
old, Thread.current[:dalli_multi] = Thread.current[:dalli_multi], true
|
49
|
-
yield
|
50
|
-
ensure
|
51
|
-
Thread.current[:dalli_multi] = old
|
61
|
+
# Get the value associated with the key.
|
62
|
+
# If a value is not found, then +nil+ is returned.
|
63
|
+
def get(key, req_options = nil)
|
64
|
+
perform(:get, key, req_options)
|
52
65
|
end
|
53
66
|
|
54
67
|
##
|
55
|
-
#
|
68
|
+
# Gat (get and touch) fetch an item and simultaneously update its expiration time.
|
69
|
+
#
|
56
70
|
# If a value is not found, then +nil+ is returned.
|
57
|
-
def
|
58
|
-
perform(:
|
71
|
+
def gat(key, ttl = nil)
|
72
|
+
perform(:gat, key, ttl_or_default(ttl))
|
73
|
+
end
|
74
|
+
|
75
|
+
##
|
76
|
+
# Touch updates expiration time for a given key.
|
77
|
+
#
|
78
|
+
# Returns true if key exists, otherwise nil.
|
79
|
+
def touch(key, ttl = nil)
|
80
|
+
resp = perform(:touch, key, ttl_or_default(ttl))
|
81
|
+
resp.nil? ? nil : true
|
82
|
+
end
|
83
|
+
|
84
|
+
##
|
85
|
+
# Get the value and CAS ID associated with the key. If a block is provided,
|
86
|
+
# value and CAS will be passed to the block.
|
87
|
+
def get_cas(key)
|
88
|
+
(value, cas) = perform(:cas, key)
|
89
|
+
return [value, cas] unless block_given?
|
90
|
+
|
91
|
+
yield value, cas
|
59
92
|
end
|
60
93
|
|
61
94
|
##
|
@@ -63,17 +96,35 @@ module Dalli
|
|
63
96
|
# If a block is given, yields key/value pairs one at a time.
|
64
97
|
# Otherwise returns a hash of { 'key' => 'value', 'key2' => 'value1' }
|
65
98
|
def get_multi(*keys)
|
66
|
-
|
99
|
+
keys.flatten!
|
100
|
+
keys.compact!
|
101
|
+
|
102
|
+
return {} if keys.empty?
|
103
|
+
|
67
104
|
if block_given?
|
68
|
-
|
105
|
+
pipelined_getter.process(keys) { |k, data| yield k, data.first }
|
69
106
|
else
|
70
|
-
|
71
|
-
|
107
|
+
{}.tap do |hash|
|
108
|
+
pipelined_getter.process(keys) { |k, data| hash[k] = data.first }
|
72
109
|
end
|
73
110
|
end
|
74
111
|
end
|
75
112
|
|
76
|
-
|
113
|
+
##
|
114
|
+
# Fetch multiple keys efficiently, including available metadata such as CAS.
|
115
|
+
# If a block is given, yields key/data pairs one a time. Data is an array:
|
116
|
+
# [value, cas_id]
|
117
|
+
# If no block is given, returns a hash of
|
118
|
+
# { 'key' => [value, cas_id] }
|
119
|
+
def get_multi_cas(*keys)
|
120
|
+
if block_given?
|
121
|
+
pipelined_getter.process(keys) { |*args| yield(*args) }
|
122
|
+
else
|
123
|
+
{}.tap do |hash|
|
124
|
+
pipelined_getter.process(keys) { |k, data| hash[k] = data }
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
77
128
|
|
78
129
|
# Fetch the value associated with the key.
|
79
130
|
# If a value is found, then it is returned.
|
@@ -83,17 +134,14 @@ module Dalli
|
|
83
134
|
# If a value is not found (or if the found value is nil and :cache_nils is false)
|
84
135
|
# and a block is given, the block will be invoked and its return value
|
85
136
|
# written to the cache and returned.
|
86
|
-
def fetch(key, ttl=nil,
|
87
|
-
|
88
|
-
val = get(key,
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
add(key, val, ttl_or_default(ttl), options)
|
95
|
-
end
|
96
|
-
val
|
137
|
+
def fetch(key, ttl = nil, req_options = nil)
|
138
|
+
req_options = req_options.nil? ? CACHE_NILS : req_options.merge(CACHE_NILS) if cache_nils
|
139
|
+
val = get(key, req_options)
|
140
|
+
return val unless block_given? && not_found?(val)
|
141
|
+
|
142
|
+
new_val = yield
|
143
|
+
add(key, new_val, ttl_or_default(ttl), req_options)
|
144
|
+
new_val
|
97
145
|
end
|
98
146
|
|
99
147
|
##
|
@@ -107,8 +155,8 @@ module Dalli
|
|
107
155
|
# - nil if the key did not exist.
|
108
156
|
# - false if the value was changed by someone else.
|
109
157
|
# - true if the value was successfully updated.
|
110
|
-
def cas(key, ttl=nil,
|
111
|
-
cas_core(key, false, ttl,
|
158
|
+
def cas(key, ttl = nil, req_options = nil, &block)
|
159
|
+
cas_core(key, false, ttl, req_options, &block)
|
112
160
|
end
|
113
161
|
|
114
162
|
##
|
@@ -118,30 +166,78 @@ module Dalli
|
|
118
166
|
# Returns:
|
119
167
|
# - false if the value was changed by someone else.
|
120
168
|
# - true if the value was successfully updated.
|
121
|
-
def cas!(key, ttl=nil,
|
122
|
-
cas_core(key, true, ttl,
|
169
|
+
def cas!(key, ttl = nil, req_options = nil, &block)
|
170
|
+
cas_core(key, true, ttl, req_options, &block)
|
123
171
|
end
|
124
172
|
|
125
|
-
|
126
|
-
|
173
|
+
##
|
174
|
+
# Turn on quiet aka noreply support for a number of
|
175
|
+
# memcached operations.
|
176
|
+
#
|
177
|
+
# All relevant operations within this block will be effectively
|
178
|
+
# pipelined as Dalli will use 'quiet' versions. The invoked methods
|
179
|
+
# will all return nil, rather than their usual response. Method
|
180
|
+
# latency will be substantially lower, as the caller will not be
|
181
|
+
# blocking on responses.
|
182
|
+
#
|
183
|
+
# Currently supports storage (set, add, replace, append, prepend),
|
184
|
+
# arithmetic (incr, decr), flush and delete operations. Use of
|
185
|
+
# unsupported operations inside a block will raise an error.
|
186
|
+
#
|
187
|
+
# Any error replies will be discarded at the end of the block, and
|
188
|
+
# Dalli client methods invoked inside the block will not
|
189
|
+
# have return values
|
190
|
+
def quiet
|
191
|
+
old = Thread.current[::Dalli::QUIET]
|
192
|
+
Thread.current[::Dalli::QUIET] = true
|
193
|
+
yield
|
194
|
+
ensure
|
195
|
+
@ring&.pipeline_consume_and_ignore_responses
|
196
|
+
Thread.current[::Dalli::QUIET] = old
|
197
|
+
end
|
198
|
+
alias multi quiet
|
199
|
+
|
200
|
+
def set(key, value, ttl = nil, req_options = nil)
|
201
|
+
set_cas(key, value, 0, ttl, req_options)
|
202
|
+
end
|
203
|
+
|
204
|
+
##
|
205
|
+
# Set the key-value pair, verifying existing CAS.
|
206
|
+
# Returns the resulting CAS value if succeeded, and falsy otherwise.
|
207
|
+
def set_cas(key, value, cas, ttl = nil, req_options = nil)
|
208
|
+
perform(:set, key, value, ttl_or_default(ttl), cas, req_options)
|
127
209
|
end
|
128
210
|
|
129
211
|
##
|
130
212
|
# Conditionally add a key/value pair, if the key does not already exist
|
131
213
|
# on the server. Returns truthy if the operation succeeded.
|
132
|
-
def add(key, value, ttl=nil,
|
133
|
-
perform(:add, key, value, ttl_or_default(ttl),
|
214
|
+
def add(key, value, ttl = nil, req_options = nil)
|
215
|
+
perform(:add, key, value, ttl_or_default(ttl), req_options)
|
134
216
|
end
|
135
217
|
|
136
218
|
##
|
137
219
|
# Conditionally add a key/value pair, only if the key already exists
|
138
220
|
# on the server. Returns truthy if the operation succeeded.
|
139
|
-
def replace(key, value, ttl=nil,
|
140
|
-
|
221
|
+
def replace(key, value, ttl = nil, req_options = nil)
|
222
|
+
replace_cas(key, value, 0, ttl, req_options)
|
223
|
+
end
|
224
|
+
|
225
|
+
##
|
226
|
+
# Conditionally add a key/value pair, verifying existing CAS, only if the
|
227
|
+
# key already exists on the server. Returns the new CAS value if the
|
228
|
+
# operation succeeded, or falsy otherwise.
|
229
|
+
def replace_cas(key, value, cas, ttl = nil, req_options = nil)
|
230
|
+
perform(:replace, key, value, ttl_or_default(ttl), cas, req_options)
|
231
|
+
end
|
232
|
+
|
233
|
+
# Delete a key/value pair, verifying existing CAS.
|
234
|
+
# Returns true if succeeded, and falsy otherwise.
|
235
|
+
def delete_cas(key, cas = 0)
|
236
|
+
perform(:delete, key, cas)
|
141
237
|
end
|
142
238
|
|
143
239
|
def delete(key)
|
144
|
-
|
240
|
+
delete_cas(key, 0)
|
145
241
|
end
|
146
242
|
|
147
243
|
##
|
@@ -158,13 +254,6 @@ module Dalli
|
|
158
254
|
perform(:prepend, key, value.to_s)
|
159
255
|
end
|
160
256
|
|
161
|
-
def flush(delay=0)
|
162
|
-
time = -delay
|
163
|
-
ring.servers.map { |s| s.request(:flush, time += delay) }
|
164
|
-
end
|
165
|
-
|
166
|
-
alias_method :flush_all, :flush
|
167
|
-
|
168
257
|
##
|
169
258
|
# Incr adds the given amount to the counter on the memcached server.
|
170
259
|
# Amt must be a positive integer value.
|
@@ -176,8 +265,11 @@ module Dalli
|
|
176
265
|
# Note that the ttl will only apply if the counter does not already
|
177
266
|
# exist. To increase an existing counter and update its TTL, use
|
178
267
|
# #cas.
|
179
|
-
|
180
|
-
|
268
|
+
#
|
269
|
+
# If the value already exists, it must have been set with raw: true
|
270
|
+
def incr(key, amt = 1, ttl = nil, default = nil)
|
271
|
+
check_positive!(amt)
|
272
|
+
|
181
273
|
perform(:incr, key, amt.to_i, ttl_or_default(ttl), default)
|
182
274
|
end
|
183
275
|
|
@@ -195,29 +287,34 @@ module Dalli
|
|
195
287
|
# Note that the ttl will only apply if the counter does not already
|
196
288
|
# exist. To decrease an existing counter and update its TTL, use
|
197
289
|
# #cas.
|
198
|
-
|
199
|
-
|
290
|
+
#
|
291
|
+
# If the value already exists, it must have been set with raw: true
|
292
|
+
def decr(key, amt = 1, ttl = nil, default = nil)
|
293
|
+
check_positive!(amt)
|
294
|
+
|
200
295
|
perform(:decr, key, amt.to_i, ttl_or_default(ttl), default)
|
201
296
|
end
|
202
297
|
|
203
298
|
##
|
204
|
-
#
|
205
|
-
#
|
206
|
-
|
207
|
-
def
|
208
|
-
|
209
|
-
resp.nil? ? nil : true
|
299
|
+
# Flush the memcached server, at 'delay' seconds in the future.
|
300
|
+
# Delay defaults to zero seconds, which means an immediate flush.
|
301
|
+
##
|
302
|
+
def flush(delay = 0)
|
303
|
+
ring.servers.map { |s| s.request(:flush, delay) }
|
210
304
|
end
|
305
|
+
alias flush_all flush
|
306
|
+
|
307
|
+
ALLOWED_STAT_KEYS = %i[items slabs settings].freeze
|
211
308
|
|
212
309
|
##
|
213
310
|
# Collect the stats for each server.
|
214
311
|
# You can optionally pass a type including :items, :slabs or :settings to get specific stats
|
215
312
|
# Returns a hash like { 'hostname:port' => { 'stat1' => 'value1', ... }, 'hostname2:port' => { ... } }
|
216
|
-
def stats(type=nil)
|
217
|
-
type = nil
|
313
|
+
def stats(type = nil)
|
314
|
+
type = nil unless ALLOWED_STAT_KEYS.include? type
|
218
315
|
values = {}
|
219
316
|
ring.servers.each do |server|
|
220
|
-
values[
|
317
|
+
values[server.name.to_s] = server.alive? ? server.request(:stats, type.to_s) : nil
|
221
318
|
end
|
222
319
|
values
|
223
320
|
end
|
@@ -230,32 +327,40 @@ module Dalli
|
|
230
327
|
end
|
231
328
|
end
|
232
329
|
|
233
|
-
##
|
234
|
-
## Make sure memcache servers are alive, or raise an Dalli::RingError
|
235
|
-
def alive!
|
236
|
-
ring.server_for_key("")
|
237
|
-
end
|
238
|
-
|
239
330
|
##
|
240
331
|
## Version of the memcache servers.
|
241
332
|
def version
|
242
333
|
values = {}
|
243
334
|
ring.servers.each do |server|
|
244
|
-
values[
|
335
|
+
values[server.name.to_s] = server.alive? ? server.request(:version) : nil
|
245
336
|
end
|
246
337
|
values
|
247
338
|
end
|
248
339
|
|
340
|
+
##
|
341
|
+
## Make sure memcache servers are alive, or raise an Dalli::RingError
|
342
|
+
def alive!
|
343
|
+
ring.server_for_key('')
|
344
|
+
end
|
345
|
+
|
249
346
|
##
|
250
347
|
# Close our connection to each server.
|
251
348
|
# If you perform another operation after this, the connections will be re-established.
|
252
349
|
def close
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
350
|
+
@ring&.close
|
351
|
+
@ring = nil
|
352
|
+
end
|
353
|
+
alias reset close
|
354
|
+
|
355
|
+
CACHE_NILS = { cache_nils: true }.freeze
|
356
|
+
|
357
|
+
def not_found?(val)
|
358
|
+
cache_nils ? val == ::Dalli::NOT_FOUND : val.nil?
|
359
|
+
end
|
360
|
+
|
361
|
+
def cache_nils
|
362
|
+
@options[:cache_nils]
|
257
363
|
end
|
258
|
-
alias_method :reset, :close
|
259
364
|
|
260
365
|
# Stub method so a bare Dalli client can pretend to be a connection pool.
|
261
366
|
def with
|
@@ -264,207 +369,76 @@ module Dalli
|
|
264
369
|
|
265
370
|
private
|
266
371
|
|
267
|
-
def
|
372
|
+
def check_positive!(amt)
|
373
|
+
raise ArgumentError, "Positive values only: #{amt}" if amt.negative?
|
374
|
+
end
|
375
|
+
|
376
|
+
def cas_core(key, always_set, ttl = nil, req_options = nil)
|
268
377
|
(value, cas) = perform(:cas, key)
|
269
|
-
value = (!value || value == 'Not found') ? nil : value
|
270
378
|
return if value.nil? && !always_set
|
379
|
+
|
271
380
|
newvalue = yield(value)
|
272
|
-
perform(:set, key, newvalue, ttl_or_default(ttl), cas,
|
381
|
+
perform(:set, key, newvalue, ttl_or_default(ttl), cas, req_options)
|
273
382
|
end
|
274
383
|
|
384
|
+
##
|
385
|
+
# Uses the argument TTL or the client-wide default. Ensures
|
386
|
+
# that the value is an integer
|
387
|
+
##
|
275
388
|
def ttl_or_default(ttl)
|
276
389
|
(ttl || @options[:expires_in]).to_i
|
277
390
|
rescue NoMethodError
|
278
391
|
raise ArgumentError, "Cannot convert ttl (#{ttl}) to an integer"
|
279
392
|
end
|
280
393
|
|
281
|
-
def
|
282
|
-
|
283
|
-
begin
|
284
|
-
ring.server_for_key(key)
|
285
|
-
rescue Dalli::RingError
|
286
|
-
Dalli.logger.debug { "unable to get key #{key}" }
|
287
|
-
nil
|
288
|
-
end
|
289
|
-
end
|
290
|
-
return groups
|
291
|
-
end
|
292
|
-
|
293
|
-
def mapped_keys(keys)
|
294
|
-
keys.flatten.map {|a| validate_key(a.to_s)}
|
295
|
-
end
|
296
|
-
|
297
|
-
def make_multi_get_requests(groups)
|
298
|
-
groups.each do |server, keys_for_server|
|
299
|
-
begin
|
300
|
-
# TODO: do this with the perform chokepoint?
|
301
|
-
# But given the fact that fetching the response doesn't take place
|
302
|
-
# in that slot it's misleading anyway. Need to move all of this method
|
303
|
-
# into perform to be meaningful
|
304
|
-
server.request(:send_multiget, keys_for_server)
|
305
|
-
rescue DalliError, NetworkError => e
|
306
|
-
Dalli.logger.debug { e.inspect }
|
307
|
-
Dalli.logger.debug { "unable to get keys for server #{server.name}" }
|
308
|
-
end
|
309
|
-
end
|
394
|
+
def ring
|
395
|
+
@ring ||= Dalli::Ring.new(@normalized_servers, protocol_implementation, @options)
|
310
396
|
end
|
311
397
|
|
312
|
-
def
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
Dalli.logger.debug { "results from this server will be missing" }
|
320
|
-
servers.delete(server)
|
321
|
-
end
|
322
|
-
end
|
323
|
-
servers
|
398
|
+
def protocol_implementation
|
399
|
+
@protocol_implementation ||= case @options[:protocol]&.to_s
|
400
|
+
when 'meta'
|
401
|
+
Dalli::Protocol::Meta
|
402
|
+
else
|
403
|
+
Dalli::Protocol::Binary
|
404
|
+
end
|
324
405
|
end
|
325
406
|
|
326
407
|
##
|
327
|
-
#
|
328
|
-
#
|
329
|
-
#
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
end
|
337
|
-
|
338
|
-
def ring
|
339
|
-
@ring ||= Dalli::Ring.new(
|
340
|
-
@servers.map do |s|
|
341
|
-
server_options = {}
|
342
|
-
if s =~ %r{\Amemcached://}
|
343
|
-
uri = URI.parse(s)
|
344
|
-
server_options[:username] = uri.user
|
345
|
-
server_options[:password] = uri.password
|
346
|
-
s = "#{uri.host}:#{uri.port}"
|
347
|
-
end
|
348
|
-
Dalli::Server.new(s, @options.merge(server_options))
|
349
|
-
end, @options
|
350
|
-
)
|
351
|
-
end
|
352
|
-
|
353
|
-
# Chokepoint method for instrumentation
|
408
|
+
# Chokepoint method for memcached methods with a key argument.
|
409
|
+
# Validates the key, resolves the key to the appropriate server
|
410
|
+
# instance, and invokes the memcached method on the appropriate
|
411
|
+
# server.
|
412
|
+
#
|
413
|
+
# This method also forces retries on network errors - when
|
414
|
+
# a particular memcached instance becomes unreachable, or the
|
415
|
+
# operational times out.
|
416
|
+
##
|
354
417
|
def perform(*all_args)
|
355
418
|
return yield if block_given?
|
356
|
-
op, key, *args = *all_args
|
357
|
-
|
358
|
-
key = key.to_s
|
359
|
-
key = validate_key(key)
|
360
|
-
begin
|
361
|
-
server = ring.server_for_key(key)
|
362
|
-
ret = server.request(op, key, *args)
|
363
|
-
ret
|
364
|
-
rescue NetworkError => e
|
365
|
-
Dalli.logger.debug { e.inspect }
|
366
|
-
Dalli.logger.debug { "retrying request with new server" }
|
367
|
-
retry
|
368
|
-
end
|
369
|
-
end
|
370
|
-
|
371
|
-
def validate_key(key)
|
372
|
-
raise ArgumentError, "key cannot be blank" if !key || key.length == 0
|
373
|
-
key = key_with_namespace(key)
|
374
|
-
if key.length > 250
|
375
|
-
max_length_before_namespace = 212 - (namespace || '').size
|
376
|
-
key = "#{key[0, max_length_before_namespace]}:md5:#{Digest::MD5.hexdigest(key)}"
|
377
|
-
end
|
378
|
-
return key
|
379
|
-
end
|
380
419
|
|
381
|
-
|
382
|
-
(ns = namespace) ? "#{ns}:#{key}" : key
|
383
|
-
end
|
420
|
+
op, key, *args = all_args
|
384
421
|
|
385
|
-
|
386
|
-
|
387
|
-
end
|
422
|
+
key = key.to_s
|
423
|
+
key = @key_manager.validate_key(key)
|
388
424
|
|
389
|
-
|
390
|
-
|
391
|
-
|
425
|
+
server = ring.server_for_key(key)
|
426
|
+
server.request(op, key, *args)
|
427
|
+
rescue NetworkError => e
|
428
|
+
Dalli.logger.debug { e.inspect }
|
429
|
+
Dalli.logger.debug { 'retrying request with new server' }
|
430
|
+
retry
|
392
431
|
end
|
393
432
|
|
394
433
|
def normalize_options(opts)
|
395
|
-
if opts[:
|
396
|
-
Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just :compress => true. Please update your configuration."
|
397
|
-
opts[:compress] = opts.delete(:compression)
|
398
|
-
end
|
399
|
-
begin
|
400
|
-
opts[:expires_in] = opts[:expires_in].to_i if opts[:expires_in]
|
401
|
-
rescue NoMethodError
|
402
|
-
raise ArgumentError, "cannot convert :expires_in => #{opts[:expires_in].inspect} to an integer"
|
403
|
-
end
|
434
|
+
opts[:expires_in] = opts[:expires_in].to_i if opts[:expires_in]
|
404
435
|
opts
|
436
|
+
rescue NoMethodError
|
437
|
+
raise ArgumentError, "cannot convert :expires_in => #{opts[:expires_in].inspect} to an integer"
|
405
438
|
end
|
406
439
|
|
407
|
-
|
408
|
-
|
409
|
-
def get_multi_yielder(keys)
|
410
|
-
perform do
|
411
|
-
return {} if keys.empty?
|
412
|
-
ring.lock do
|
413
|
-
begin
|
414
|
-
groups = groups_for_keys(keys)
|
415
|
-
if unfound_keys = groups.delete(nil)
|
416
|
-
Dalli.logger.debug { "unable to get keys for #{unfound_keys.length} keys because no matching server was found" }
|
417
|
-
end
|
418
|
-
make_multi_get_requests(groups)
|
419
|
-
|
420
|
-
servers = groups.keys
|
421
|
-
return if servers.empty?
|
422
|
-
servers = perform_multi_response_start(servers)
|
423
|
-
|
424
|
-
start = Time.now
|
425
|
-
while true
|
426
|
-
# remove any dead servers
|
427
|
-
servers.delete_if { |s| s.sock.nil? }
|
428
|
-
break if servers.empty?
|
429
|
-
|
430
|
-
# calculate remaining timeout
|
431
|
-
elapsed = Time.now - start
|
432
|
-
timeout = servers.first.options[:socket_timeout]
|
433
|
-
time_left = (elapsed > timeout) ? 0 : timeout - elapsed
|
434
|
-
|
435
|
-
sockets = servers.map(&:sock)
|
436
|
-
readable, _ = IO.select(sockets, nil, nil, time_left)
|
437
|
-
|
438
|
-
if readable.nil?
|
439
|
-
# no response within timeout; abort pending connections
|
440
|
-
servers.each do |server|
|
441
|
-
Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
|
442
|
-
server.multi_response_abort
|
443
|
-
end
|
444
|
-
break
|
445
|
-
|
446
|
-
else
|
447
|
-
readable.each do |sock|
|
448
|
-
server = sock.server
|
449
|
-
|
450
|
-
begin
|
451
|
-
server.multi_response_nonblock.each_pair do |key, value_list|
|
452
|
-
yield key_without_namespace(key), value_list
|
453
|
-
end
|
454
|
-
|
455
|
-
if server.multi_response_completed?
|
456
|
-
servers.delete(server)
|
457
|
-
end
|
458
|
-
rescue NetworkError
|
459
|
-
servers.delete(server)
|
460
|
-
end
|
461
|
-
end
|
462
|
-
end
|
463
|
-
end
|
464
|
-
end
|
465
|
-
end
|
466
|
-
end
|
440
|
+
def pipelined_getter
|
441
|
+
PipelinedGetter.new(ring, @key_manager)
|
467
442
|
end
|
468
|
-
|
469
443
|
end
|
470
444
|
end
|
data/lib/dalli/compressor.rb
CHANGED
@@ -1,7 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'zlib'
|
2
4
|
require 'stringio'
|
3
5
|
|
4
6
|
module Dalli
|
7
|
+
##
|
8
|
+
# Default compressor used by Dalli, that uses
|
9
|
+
# Zlib DEFLATE to compress data.
|
10
|
+
##
|
5
11
|
class Compressor
|
6
12
|
def self.compress(data)
|
7
13
|
Zlib::Deflate.deflate(data)
|
@@ -12,9 +18,14 @@ module Dalli
|
|
12
18
|
end
|
13
19
|
end
|
14
20
|
|
21
|
+
##
|
22
|
+
# Alternate compressor for Dalli, that uses
|
23
|
+
# Gzip. Gzip adds a checksum to each compressed
|
24
|
+
# entry.
|
25
|
+
##
|
15
26
|
class GzipCompressor
|
16
27
|
def self.compress(data)
|
17
|
-
io = StringIO.new(
|
28
|
+
io = StringIO.new(+'', 'w')
|
18
29
|
gz = Zlib::GzipWriter.new(io)
|
19
30
|
gz.write(data)
|
20
31
|
gz.close
|
@@ -22,7 +33,7 @@ module Dalli
|
|
22
33
|
end
|
23
34
|
|
24
35
|
def self.decompress(data)
|
25
|
-
io = StringIO.new(data,
|
36
|
+
io = StringIO.new(data, 'rb')
|
26
37
|
Zlib::GzipReader.new(io).read
|
27
38
|
end
|
28
39
|
end
|