dalli 2.7.10 → 3.2.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of dalli might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Gemfile +5 -1
- data/History.md +148 -2
- data/README.md +26 -223
- data/lib/dalli/cas/client.rb +1 -57
- data/lib/dalli/client.rb +226 -258
- data/lib/dalli/compressor.rb +12 -2
- data/lib/dalli/key_manager.rb +113 -0
- data/lib/dalli/options.rb +6 -7
- data/lib/dalli/pipelined_getter.rb +177 -0
- data/lib/dalli/protocol/base.rb +241 -0
- data/lib/dalli/protocol/binary/request_formatter.rb +117 -0
- data/lib/dalli/protocol/binary/response_header.rb +36 -0
- data/lib/dalli/protocol/binary/response_processor.rb +239 -0
- data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
- data/lib/dalli/protocol/binary.rb +173 -0
- data/lib/dalli/protocol/connection_manager.rb +252 -0
- data/lib/dalli/protocol/meta/key_regularizer.rb +31 -0
- data/lib/dalli/protocol/meta/request_formatter.rb +108 -0
- data/lib/dalli/protocol/meta/response_processor.rb +211 -0
- data/lib/dalli/protocol/meta.rb +177 -0
- data/lib/dalli/protocol/response_buffer.rb +54 -0
- data/lib/dalli/protocol/server_config_parser.rb +84 -0
- data/lib/dalli/protocol/ttl_sanitizer.rb +45 -0
- data/lib/dalli/protocol/value_compressor.rb +85 -0
- data/lib/dalli/protocol/value_marshaller.rb +59 -0
- data/lib/dalli/protocol/value_serializer.rb +91 -0
- data/lib/dalli/protocol.rb +8 -0
- data/lib/dalli/ring.rb +94 -83
- data/lib/dalli/server.rb +3 -746
- data/lib/dalli/servers_arg_normalizer.rb +54 -0
- data/lib/dalli/socket.rb +117 -137
- data/lib/dalli/version.rb +4 -1
- data/lib/dalli.rb +43 -15
- data/lib/rack/session/dalli.rb +103 -94
- metadata +64 -26
- data/lib/action_dispatch/middleware/session/dalli_store.rb +0 -82
- data/lib/active_support/cache/dalli_store.rb +0 -435
- data/lib/dalli/railtie.rb +0 -8
data/lib/dalli/server.rb
CHANGED
@@ -1,749 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'socket'
|
3
|
-
require 'timeout'
|
4
2
|
|
5
|
-
module Dalli
|
6
|
-
|
7
|
-
|
8
|
-
attr_accessor :port
|
9
|
-
attr_accessor :weight
|
10
|
-
attr_accessor :options
|
11
|
-
attr_reader :sock
|
12
|
-
attr_reader :socket_type # possible values: :unix, :tcp
|
13
|
-
|
14
|
-
DEFAULT_PORT = 11211
|
15
|
-
DEFAULT_WEIGHT = 1
|
16
|
-
DEFAULTS = {
|
17
|
-
# seconds between trying to contact a remote server
|
18
|
-
:down_retry_delay => 60,
|
19
|
-
# connect/read/write timeout for socket operations
|
20
|
-
:socket_timeout => 0.5,
|
21
|
-
# times a socket operation may fail before considering the server dead
|
22
|
-
:socket_max_failures => 2,
|
23
|
-
# amount of time to sleep between retries when a failure occurs
|
24
|
-
:socket_failure_delay => 0.01,
|
25
|
-
# max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
|
26
|
-
:value_max_bytes => 1024 * 1024,
|
27
|
-
# surpassing value_max_bytes either warns (false) or throws (true)
|
28
|
-
:error_when_over_max_size => false,
|
29
|
-
:compressor => Compressor,
|
30
|
-
# min byte size to attempt compression
|
31
|
-
:compression_min_size => 1024,
|
32
|
-
# max byte size for compression
|
33
|
-
:compression_max_size => false,
|
34
|
-
:serializer => Marshal,
|
35
|
-
:username => nil,
|
36
|
-
:password => nil,
|
37
|
-
:keepalive => true,
|
38
|
-
# max byte size for SO_SNDBUF
|
39
|
-
:sndbuf => nil,
|
40
|
-
# max byte size for SO_RCVBUF
|
41
|
-
:rcvbuf => nil
|
42
|
-
}
|
43
|
-
|
44
|
-
def initialize(attribs, options = {})
|
45
|
-
@hostname, @port, @weight, @socket_type = parse_hostname(attribs)
|
46
|
-
@fail_count = 0
|
47
|
-
@down_at = nil
|
48
|
-
@last_down_at = nil
|
49
|
-
@options = DEFAULTS.merge(options)
|
50
|
-
@sock = nil
|
51
|
-
@msg = nil
|
52
|
-
@error = nil
|
53
|
-
@pid = nil
|
54
|
-
@inprogress = nil
|
55
|
-
end
|
56
|
-
|
57
|
-
def name
|
58
|
-
if socket_type == :unix
|
59
|
-
hostname
|
60
|
-
else
|
61
|
-
"#{hostname}:#{port}"
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
# Chokepoint method for instrumentation
|
66
|
-
def request(op, *args)
|
67
|
-
verify_state
|
68
|
-
raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}. If you are sure it is running, ensure memcached version is > 1.4." unless alive?
|
69
|
-
begin
|
70
|
-
send(op, *args)
|
71
|
-
rescue Dalli::MarshalError => ex
|
72
|
-
Dalli.logger.error "Marshalling error for key '#{args.first}': #{ex.message}"
|
73
|
-
Dalli.logger.error "You are trying to cache a Ruby object which cannot be serialized to memcached."
|
74
|
-
Dalli.logger.error ex.backtrace.join("\n\t")
|
75
|
-
false
|
76
|
-
rescue Dalli::DalliError, Dalli::NetworkError, Dalli::ValueOverMaxSize, Timeout::Error
|
77
|
-
raise
|
78
|
-
rescue => ex
|
79
|
-
Dalli.logger.error "Unexpected exception during Dalli request: #{ex.class.name}: #{ex.message}"
|
80
|
-
Dalli.logger.error ex.backtrace.join("\n\t")
|
81
|
-
down!
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
def alive?
|
86
|
-
return true if @sock
|
87
|
-
|
88
|
-
if @last_down_at && @last_down_at + options[:down_retry_delay] >= Time.now
|
89
|
-
time = @last_down_at + options[:down_retry_delay] - Time.now
|
90
|
-
Dalli.logger.debug { "down_retry_delay not reached for #{name} (%.3f seconds left)" % time }
|
91
|
-
return false
|
92
|
-
end
|
93
|
-
|
94
|
-
connect
|
95
|
-
!!@sock
|
96
|
-
rescue Dalli::NetworkError
|
97
|
-
false
|
98
|
-
end
|
99
|
-
|
100
|
-
def close
|
101
|
-
return unless @sock
|
102
|
-
@sock.close rescue nil
|
103
|
-
@sock = nil
|
104
|
-
@pid = nil
|
105
|
-
@inprogress = false
|
106
|
-
end
|
107
|
-
|
108
|
-
def lock!
|
109
|
-
end
|
110
|
-
|
111
|
-
def unlock!
|
112
|
-
end
|
113
|
-
|
114
|
-
def serializer
|
115
|
-
@options[:serializer]
|
116
|
-
end
|
117
|
-
|
118
|
-
def compressor
|
119
|
-
@options[:compressor]
|
120
|
-
end
|
121
|
-
|
122
|
-
# Start reading key/value pairs from this connection. This is usually called
|
123
|
-
# after a series of GETKQ commands. A NOOP is sent, and the server begins
|
124
|
-
# flushing responses for kv pairs that were found.
|
125
|
-
#
|
126
|
-
# Returns nothing.
|
127
|
-
def multi_response_start
|
128
|
-
verify_state
|
129
|
-
write_noop
|
130
|
-
@multi_buffer = String.new('')
|
131
|
-
@position = 0
|
132
|
-
@inprogress = true
|
133
|
-
end
|
134
|
-
|
135
|
-
# Did the last call to #multi_response_start complete successfully?
|
136
|
-
def multi_response_completed?
|
137
|
-
@multi_buffer.nil?
|
138
|
-
end
|
139
|
-
|
140
|
-
# Attempt to receive and parse as many key/value pairs as possible
|
141
|
-
# from this server. After #multi_response_start, this should be invoked
|
142
|
-
# repeatedly whenever this server's socket is readable until
|
143
|
-
# #multi_response_completed?.
|
144
|
-
#
|
145
|
-
# Returns a Hash of kv pairs received.
|
146
|
-
def multi_response_nonblock
|
147
|
-
raise 'multi_response has completed' if @multi_buffer.nil?
|
148
|
-
|
149
|
-
@multi_buffer << @sock.read_available
|
150
|
-
buf = @multi_buffer
|
151
|
-
pos = @position
|
152
|
-
values = {}
|
153
|
-
|
154
|
-
while buf.bytesize - pos >= 24
|
155
|
-
header = buf.slice(pos, 24)
|
156
|
-
(key_length, _, body_length, cas) = header.unpack(KV_HEADER)
|
157
|
-
|
158
|
-
if key_length == 0
|
159
|
-
# all done!
|
160
|
-
@multi_buffer = nil
|
161
|
-
@position = nil
|
162
|
-
@inprogress = false
|
163
|
-
break
|
164
|
-
|
165
|
-
elsif buf.bytesize - pos >= 24 + body_length
|
166
|
-
flags = buf.slice(pos + 24, 4).unpack('N')[0]
|
167
|
-
key = buf.slice(pos + 24 + 4, key_length)
|
168
|
-
value = buf.slice(pos + 24 + 4 + key_length, body_length - key_length - 4) if body_length - key_length - 4 > 0
|
169
|
-
|
170
|
-
pos = pos + 24 + body_length
|
171
|
-
|
172
|
-
begin
|
173
|
-
values[key] = [deserialize(value, flags), cas]
|
174
|
-
rescue DalliError
|
175
|
-
end
|
176
|
-
|
177
|
-
else
|
178
|
-
# not enough data yet, wait for more
|
179
|
-
break
|
180
|
-
end
|
181
|
-
end
|
182
|
-
@position = pos
|
183
|
-
|
184
|
-
values
|
185
|
-
rescue SystemCallError, Timeout::Error, EOFError => e
|
186
|
-
failure!(e)
|
187
|
-
end
|
188
|
-
|
189
|
-
# Abort an earlier #multi_response_start. Used to signal an external
|
190
|
-
# timeout. The underlying socket is disconnected, and the exception is
|
191
|
-
# swallowed.
|
192
|
-
#
|
193
|
-
# Returns nothing.
|
194
|
-
def multi_response_abort
|
195
|
-
@multi_buffer = nil
|
196
|
-
@position = nil
|
197
|
-
@inprogress = false
|
198
|
-
failure!(RuntimeError.new('External timeout'))
|
199
|
-
rescue NetworkError
|
200
|
-
true
|
201
|
-
end
|
202
|
-
|
203
|
-
# NOTE: Additional public methods should be overridden in Dalli::Threadsafe
|
204
|
-
|
205
|
-
private
|
206
|
-
|
207
|
-
def verify_state
|
208
|
-
failure!(RuntimeError.new('Already writing to socket')) if @inprogress
|
209
|
-
if @pid && @pid != Process.pid
|
210
|
-
message = 'Fork detected, re-connecting child process...'
|
211
|
-
Dalli.logger.info { message }
|
212
|
-
reconnect! message
|
213
|
-
end
|
214
|
-
end
|
215
|
-
|
216
|
-
def reconnect!(message)
|
217
|
-
close
|
218
|
-
sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
|
219
|
-
raise Dalli::NetworkError, message
|
220
|
-
end
|
221
|
-
|
222
|
-
def failure!(exception)
|
223
|
-
message = "#{name} failed (count: #{@fail_count}) #{exception.class}: #{exception.message}"
|
224
|
-
Dalli.logger.warn { message }
|
225
|
-
|
226
|
-
@fail_count += 1
|
227
|
-
if @fail_count >= options[:socket_max_failures]
|
228
|
-
down!
|
229
|
-
else
|
230
|
-
reconnect! 'Socket operation failed, retrying...'
|
231
|
-
end
|
232
|
-
end
|
233
|
-
|
234
|
-
def down!
|
235
|
-
close
|
236
|
-
|
237
|
-
@last_down_at = Time.now
|
238
|
-
|
239
|
-
if @down_at
|
240
|
-
time = Time.now - @down_at
|
241
|
-
Dalli.logger.debug { "#{name} is still down (for %.3f seconds now)" % time }
|
242
|
-
else
|
243
|
-
@down_at = @last_down_at
|
244
|
-
Dalli.logger.warn { "#{name} is down" }
|
245
|
-
end
|
246
|
-
|
247
|
-
@error = $! && $!.class.name
|
248
|
-
@msg = @msg || ($! && $!.message && !$!.message.empty? && $!.message)
|
249
|
-
raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
|
250
|
-
end
|
251
|
-
|
252
|
-
def up!
|
253
|
-
if @down_at
|
254
|
-
time = Time.now - @down_at
|
255
|
-
Dalli.logger.warn { "#{name} is back (downtime was %.3f seconds)" % time }
|
256
|
-
end
|
257
|
-
|
258
|
-
@fail_count = 0
|
259
|
-
@down_at = nil
|
260
|
-
@last_down_at = nil
|
261
|
-
@msg = nil
|
262
|
-
@error = nil
|
263
|
-
end
|
264
|
-
|
265
|
-
def multi?
|
266
|
-
Thread.current[:dalli_multi]
|
267
|
-
end
|
268
|
-
|
269
|
-
def get(key, options=nil)
|
270
|
-
req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
|
271
|
-
write(req)
|
272
|
-
generic_response(true, !!(options && options.is_a?(Hash) && options[:cache_nils]))
|
273
|
-
end
|
274
|
-
|
275
|
-
def send_multiget(keys)
|
276
|
-
req = String.new("")
|
277
|
-
keys.each do |key|
|
278
|
-
req << [REQUEST, OPCODES[:getkq], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:getkq])
|
279
|
-
end
|
280
|
-
# Could send noop here instead of in multi_response_start
|
281
|
-
write(req)
|
282
|
-
end
|
283
|
-
|
284
|
-
def set(key, value, ttl, cas, options)
|
285
|
-
(value, flags) = serialize(key, value, options)
|
286
|
-
ttl = sanitize_ttl(ttl)
|
287
|
-
|
288
|
-
guard_max_value(key, value) do
|
289
|
-
req = [REQUEST, OPCODES[multi? ? :setq : :set], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:set])
|
290
|
-
write(req)
|
291
|
-
cas_response unless multi?
|
292
|
-
end
|
293
|
-
end
|
294
|
-
|
295
|
-
def add(key, value, ttl, options)
|
296
|
-
(value, flags) = serialize(key, value, options)
|
297
|
-
ttl = sanitize_ttl(ttl)
|
298
|
-
|
299
|
-
guard_max_value(key, value) do
|
300
|
-
req = [REQUEST, OPCODES[multi? ? :addq : :add], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:add])
|
301
|
-
write(req)
|
302
|
-
cas_response unless multi?
|
303
|
-
end
|
304
|
-
end
|
305
|
-
|
306
|
-
def replace(key, value, ttl, cas, options)
|
307
|
-
(value, flags) = serialize(key, value, options)
|
308
|
-
ttl = sanitize_ttl(ttl)
|
309
|
-
|
310
|
-
guard_max_value(key, value) do
|
311
|
-
req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:replace])
|
312
|
-
write(req)
|
313
|
-
cas_response unless multi?
|
314
|
-
end
|
315
|
-
end
|
316
|
-
|
317
|
-
def delete(key, cas)
|
318
|
-
req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, cas, key].pack(FORMAT[:delete])
|
319
|
-
write(req)
|
320
|
-
generic_response unless multi?
|
321
|
-
end
|
322
|
-
|
323
|
-
def flush(ttl)
|
324
|
-
req = [REQUEST, OPCODES[:flush], 0, 4, 0, 0, 4, 0, 0, 0].pack(FORMAT[:flush])
|
325
|
-
write(req)
|
326
|
-
generic_response
|
327
|
-
end
|
328
|
-
|
329
|
-
def decr_incr(opcode, key, count, ttl, default)
|
330
|
-
expiry = default ? sanitize_ttl(ttl) : 0xFFFFFFFF
|
331
|
-
default ||= 0
|
332
|
-
(h, l) = split(count)
|
333
|
-
(dh, dl) = split(default)
|
334
|
-
req = [REQUEST, OPCODES[opcode], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[opcode])
|
335
|
-
write(req)
|
336
|
-
body = generic_response
|
337
|
-
body ? body.unpack('Q>').first : body
|
338
|
-
end
|
339
|
-
|
340
|
-
def decr(key, count, ttl, default)
|
341
|
-
decr_incr :decr, key, count, ttl, default
|
342
|
-
end
|
343
|
-
|
344
|
-
def incr(key, count, ttl, default)
|
345
|
-
decr_incr :incr, key, count, ttl, default
|
346
|
-
end
|
347
|
-
|
348
|
-
def write_append_prepend(opcode, key, value)
|
349
|
-
write_generic [REQUEST, OPCODES[opcode], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[opcode])
|
350
|
-
end
|
351
|
-
|
352
|
-
def write_generic(bytes)
|
353
|
-
write(bytes)
|
354
|
-
generic_response
|
355
|
-
end
|
356
|
-
|
357
|
-
def write_noop
|
358
|
-
req = [REQUEST, OPCODES[:noop], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
|
359
|
-
write(req)
|
360
|
-
end
|
361
|
-
|
362
|
-
# Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
|
363
|
-
# We need to read all the responses at once.
|
364
|
-
def noop
|
365
|
-
write_noop
|
366
|
-
multi_response
|
367
|
-
end
|
368
|
-
|
369
|
-
def append(key, value)
|
370
|
-
write_append_prepend :append, key, value
|
371
|
-
end
|
372
|
-
|
373
|
-
def prepend(key, value)
|
374
|
-
write_append_prepend :prepend, key, value
|
375
|
-
end
|
376
|
-
|
377
|
-
def stats(info='')
|
378
|
-
req = [REQUEST, OPCODES[:stat], info.bytesize, 0, 0, 0, info.bytesize, 0, 0, info].pack(FORMAT[:stat])
|
379
|
-
write(req)
|
380
|
-
keyvalue_response
|
381
|
-
end
|
382
|
-
|
383
|
-
def reset_stats
|
384
|
-
write_generic [REQUEST, OPCODES[:stat], 'reset'.bytesize, 0, 0, 0, 'reset'.bytesize, 0, 0, 'reset'].pack(FORMAT[:stat])
|
385
|
-
end
|
386
|
-
|
387
|
-
def cas(key)
|
388
|
-
req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
|
389
|
-
write(req)
|
390
|
-
data_cas_response
|
391
|
-
end
|
392
|
-
|
393
|
-
def version
|
394
|
-
write_generic [REQUEST, OPCODES[:version], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
|
395
|
-
end
|
396
|
-
|
397
|
-
def touch(key, ttl)
|
398
|
-
ttl = sanitize_ttl(ttl)
|
399
|
-
write_generic [REQUEST, OPCODES[:touch], key.bytesize, 4, 0, 0, key.bytesize + 4, 0, 0, ttl, key].pack(FORMAT[:touch])
|
400
|
-
end
|
401
|
-
|
402
|
-
# http://www.hjp.at/zettel/m/memcached_flags.rxml
|
403
|
-
# Looks like most clients use bit 0 to indicate native language serialization
|
404
|
-
# and bit 1 to indicate gzip compression.
|
405
|
-
FLAG_SERIALIZED = 0x1
|
406
|
-
FLAG_COMPRESSED = 0x2
|
407
|
-
|
408
|
-
def serialize(key, value, options=nil)
|
409
|
-
marshalled = false
|
410
|
-
value = unless options && options[:raw]
|
411
|
-
marshalled = true
|
412
|
-
begin
|
413
|
-
self.serializer.dump(value)
|
414
|
-
rescue Timeout::Error => e
|
415
|
-
raise e
|
416
|
-
rescue => ex
|
417
|
-
# Marshalling can throw several different types of generic Ruby exceptions.
|
418
|
-
# Convert to a specific exception so we can special case it higher up the stack.
|
419
|
-
exc = Dalli::MarshalError.new(ex.message)
|
420
|
-
exc.set_backtrace ex.backtrace
|
421
|
-
raise exc
|
422
|
-
end
|
423
|
-
else
|
424
|
-
value.to_s
|
425
|
-
end
|
426
|
-
compressed = false
|
427
|
-
set_compress_option = true if options && options[:compress]
|
428
|
-
if (@options[:compress] || set_compress_option) && value.bytesize >= @options[:compression_min_size] &&
|
429
|
-
(!@options[:compression_max_size] || value.bytesize <= @options[:compression_max_size])
|
430
|
-
value = self.compressor.compress(value)
|
431
|
-
compressed = true
|
432
|
-
end
|
433
|
-
|
434
|
-
flags = 0
|
435
|
-
flags |= FLAG_COMPRESSED if compressed
|
436
|
-
flags |= FLAG_SERIALIZED if marshalled
|
437
|
-
[value, flags]
|
438
|
-
end
|
439
|
-
|
440
|
-
def deserialize(value, flags)
|
441
|
-
value = self.compressor.decompress(value) if (flags & FLAG_COMPRESSED) != 0
|
442
|
-
value = self.serializer.load(value) if (flags & FLAG_SERIALIZED) != 0
|
443
|
-
value
|
444
|
-
rescue TypeError
|
445
|
-
raise if $!.message !~ /needs to have method `_load'|exception class\/object expected|instance of IO needed|incompatible marshal file format/
|
446
|
-
raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
|
447
|
-
rescue ArgumentError
|
448
|
-
raise if $!.message !~ /undefined class|marshal data too short/
|
449
|
-
raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
|
450
|
-
rescue Zlib::Error
|
451
|
-
raise UnmarshalError, "Unable to uncompress value: #{$!.message}"
|
452
|
-
end
|
453
|
-
|
454
|
-
def data_cas_response
|
455
|
-
(extras, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
|
456
|
-
data = read(count) if count > 0
|
457
|
-
if status == 1
|
458
|
-
nil
|
459
|
-
elsif status != 0
|
460
|
-
raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
|
461
|
-
elsif data
|
462
|
-
flags = data[0...extras].unpack('N')[0]
|
463
|
-
value = data[extras..-1]
|
464
|
-
data = deserialize(value, flags)
|
465
|
-
end
|
466
|
-
[data, cas]
|
467
|
-
end
|
468
|
-
|
469
|
-
CAS_HEADER = '@4CCnNNQ'
|
470
|
-
NORMAL_HEADER = '@4CCnN'
|
471
|
-
KV_HEADER = '@2n@6nN@16Q'
|
472
|
-
|
473
|
-
def guard_max_value(key, value)
|
474
|
-
if value.bytesize <= @options[:value_max_bytes]
|
475
|
-
yield
|
476
|
-
else
|
477
|
-
message = "Value for #{key} over max size: #{@options[:value_max_bytes]} <= #{value.bytesize}"
|
478
|
-
raise Dalli::ValueOverMaxSize, message if @options[:error_when_over_max_size]
|
479
|
-
|
480
|
-
Dalli.logger.error "#{message} - this value may be truncated by memcached"
|
481
|
-
false
|
482
|
-
end
|
483
|
-
end
|
484
|
-
|
485
|
-
# https://github.com/memcached/memcached/blob/master/doc/protocol.txt#L79
|
486
|
-
# > An expiration time, in seconds. Can be up to 30 days. After 30 days, is treated as a unix timestamp of an exact date.
|
487
|
-
MAX_ACCEPTABLE_EXPIRATION_INTERVAL = 30*24*60*60 # 30 days
|
488
|
-
def sanitize_ttl(ttl)
|
489
|
-
ttl_as_i = ttl.to_i
|
490
|
-
return ttl_as_i if ttl_as_i <= MAX_ACCEPTABLE_EXPIRATION_INTERVAL
|
491
|
-
now = Time.now.to_i
|
492
|
-
return ttl_as_i if ttl_as_i > now # already a timestamp
|
493
|
-
Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached, converting to an expiration timestamp"
|
494
|
-
now + ttl_as_i
|
495
|
-
end
|
496
|
-
|
497
|
-
# Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
|
498
|
-
class NilObject; end
|
499
|
-
NOT_FOUND = NilObject.new
|
500
|
-
|
501
|
-
def generic_response(unpack=false, cache_nils=false)
|
502
|
-
(extras, _, status, count) = read_header.unpack(NORMAL_HEADER)
|
503
|
-
data = read(count) if count > 0
|
504
|
-
if status == 1
|
505
|
-
cache_nils ? NOT_FOUND : nil
|
506
|
-
elsif status == 2 || status == 5
|
507
|
-
false # Not stored, normal status for add operation
|
508
|
-
elsif status != 0
|
509
|
-
raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
|
510
|
-
elsif data
|
511
|
-
flags = data[0...extras].unpack('N')[0]
|
512
|
-
value = data[extras..-1]
|
513
|
-
unpack ? deserialize(value, flags) : value
|
514
|
-
else
|
515
|
-
true
|
516
|
-
end
|
517
|
-
end
|
518
|
-
|
519
|
-
def cas_response
|
520
|
-
(_, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
|
521
|
-
read(count) if count > 0 # this is potential data that we don't care about
|
522
|
-
if status == 1
|
523
|
-
nil
|
524
|
-
elsif status == 2 || status == 5
|
525
|
-
false # Not stored, normal status for add operation
|
526
|
-
elsif status != 0
|
527
|
-
raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
|
528
|
-
else
|
529
|
-
cas
|
530
|
-
end
|
531
|
-
end
|
532
|
-
|
533
|
-
def keyvalue_response
|
534
|
-
hash = {}
|
535
|
-
while true
|
536
|
-
(key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
|
537
|
-
return hash if key_length == 0
|
538
|
-
key = read(key_length)
|
539
|
-
value = read(body_length - key_length) if body_length - key_length > 0
|
540
|
-
hash[key] = value
|
541
|
-
end
|
542
|
-
end
|
543
|
-
|
544
|
-
def multi_response
|
545
|
-
hash = {}
|
546
|
-
while true
|
547
|
-
(key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
|
548
|
-
return hash if key_length == 0
|
549
|
-
flags = read(4).unpack('N')[0]
|
550
|
-
key = read(key_length)
|
551
|
-
value = read(body_length - key_length - 4) if body_length - key_length - 4 > 0
|
552
|
-
hash[key] = deserialize(value, flags)
|
553
|
-
end
|
554
|
-
end
|
555
|
-
|
556
|
-
def write(bytes)
|
557
|
-
begin
|
558
|
-
@inprogress = true
|
559
|
-
result = @sock.write(bytes)
|
560
|
-
@inprogress = false
|
561
|
-
result
|
562
|
-
rescue SystemCallError, Timeout::Error => e
|
563
|
-
failure!(e)
|
564
|
-
end
|
565
|
-
end
|
566
|
-
|
567
|
-
def read(count)
|
568
|
-
begin
|
569
|
-
@inprogress = true
|
570
|
-
data = @sock.readfull(count)
|
571
|
-
@inprogress = false
|
572
|
-
data
|
573
|
-
rescue SystemCallError, Timeout::Error, EOFError => e
|
574
|
-
failure!(e)
|
575
|
-
end
|
576
|
-
end
|
577
|
-
|
578
|
-
def read_header
|
579
|
-
read(24) || raise(Dalli::NetworkError, 'No response')
|
580
|
-
end
|
581
|
-
|
582
|
-
def connect
|
583
|
-
Dalli.logger.debug { "Dalli::Server#connect #{name}" }
|
584
|
-
|
585
|
-
begin
|
586
|
-
@pid = Process.pid
|
587
|
-
if socket_type == :unix
|
588
|
-
@sock = KSocket::UNIX.open(hostname, self, options)
|
589
|
-
else
|
590
|
-
@sock = KSocket::TCP.open(hostname, port, self, options)
|
591
|
-
end
|
592
|
-
sasl_authentication if need_auth?
|
593
|
-
@version = version # trigger actual connect
|
594
|
-
up!
|
595
|
-
rescue Dalli::DalliError # SASL auth failure
|
596
|
-
raise
|
597
|
-
rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
|
598
|
-
# SocketError = DNS resolution failure
|
599
|
-
failure!(e)
|
600
|
-
end
|
601
|
-
end
|
602
|
-
|
603
|
-
def split(n)
|
604
|
-
[n >> 32, 0xFFFFFFFF & n]
|
605
|
-
end
|
606
|
-
|
607
|
-
REQUEST = 0x80
|
608
|
-
RESPONSE = 0x81
|
609
|
-
|
610
|
-
# Response codes taken from:
|
611
|
-
# https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
|
612
|
-
RESPONSE_CODES = {
|
613
|
-
0 => 'No error',
|
614
|
-
1 => 'Key not found',
|
615
|
-
2 => 'Key exists',
|
616
|
-
3 => 'Value too large',
|
617
|
-
4 => 'Invalid arguments',
|
618
|
-
5 => 'Item not stored',
|
619
|
-
6 => 'Incr/decr on a non-numeric value',
|
620
|
-
7 => 'The vbucket belongs to another server',
|
621
|
-
8 => 'Authentication error',
|
622
|
-
9 => 'Authentication continue',
|
623
|
-
0x20 => 'Authentication required',
|
624
|
-
0x81 => 'Unknown command',
|
625
|
-
0x82 => 'Out of memory',
|
626
|
-
0x83 => 'Not supported',
|
627
|
-
0x84 => 'Internal error',
|
628
|
-
0x85 => 'Busy',
|
629
|
-
0x86 => 'Temporary failure'
|
630
|
-
}
|
631
|
-
|
632
|
-
OPCODES = {
|
633
|
-
:get => 0x00,
|
634
|
-
:set => 0x01,
|
635
|
-
:add => 0x02,
|
636
|
-
:replace => 0x03,
|
637
|
-
:delete => 0x04,
|
638
|
-
:incr => 0x05,
|
639
|
-
:decr => 0x06,
|
640
|
-
:flush => 0x08,
|
641
|
-
:noop => 0x0A,
|
642
|
-
:version => 0x0B,
|
643
|
-
:getkq => 0x0D,
|
644
|
-
:append => 0x0E,
|
645
|
-
:prepend => 0x0F,
|
646
|
-
:stat => 0x10,
|
647
|
-
:setq => 0x11,
|
648
|
-
:addq => 0x12,
|
649
|
-
:replaceq => 0x13,
|
650
|
-
:deleteq => 0x14,
|
651
|
-
:incrq => 0x15,
|
652
|
-
:decrq => 0x16,
|
653
|
-
:auth_negotiation => 0x20,
|
654
|
-
:auth_request => 0x21,
|
655
|
-
:auth_continue => 0x22,
|
656
|
-
:touch => 0x1C,
|
657
|
-
}
|
658
|
-
|
659
|
-
HEADER = "CCnCCnNNQ"
|
660
|
-
OP_FORMAT = {
|
661
|
-
:get => 'a*',
|
662
|
-
:set => 'NNa*a*',
|
663
|
-
:add => 'NNa*a*',
|
664
|
-
:replace => 'NNa*a*',
|
665
|
-
:delete => 'a*',
|
666
|
-
:incr => 'NNNNNa*',
|
667
|
-
:decr => 'NNNNNa*',
|
668
|
-
:flush => 'N',
|
669
|
-
:noop => '',
|
670
|
-
:getkq => 'a*',
|
671
|
-
:version => '',
|
672
|
-
:stat => 'a*',
|
673
|
-
:append => 'a*a*',
|
674
|
-
:prepend => 'a*a*',
|
675
|
-
:auth_request => 'a*a*',
|
676
|
-
:auth_continue => 'a*a*',
|
677
|
-
:touch => 'Na*',
|
678
|
-
}
|
679
|
-
FORMAT = OP_FORMAT.inject({}) { |memo, (k, v)| memo[k] = HEADER + v; memo }
|
680
|
-
|
681
|
-
|
682
|
-
#######
|
683
|
-
# SASL authentication support for NorthScale
|
684
|
-
#######
|
685
|
-
|
686
|
-
def need_auth?
|
687
|
-
@options[:username] || ENV['MEMCACHE_USERNAME']
|
688
|
-
end
|
689
|
-
|
690
|
-
def username
|
691
|
-
@options[:username] || ENV['MEMCACHE_USERNAME']
|
692
|
-
end
|
693
|
-
|
694
|
-
def password
|
695
|
-
@options[:password] || ENV['MEMCACHE_PASSWORD']
|
696
|
-
end
|
697
|
-
|
698
|
-
def sasl_authentication
|
699
|
-
Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
|
700
|
-
|
701
|
-
# negotiate
|
702
|
-
req = [REQUEST, OPCODES[:auth_negotiation], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
|
703
|
-
write(req)
|
704
|
-
|
705
|
-
(extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
|
706
|
-
raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
|
707
|
-
content = read(count).gsub(/\u0000/, ' ')
|
708
|
-
return (Dalli.logger.debug("Authentication not required/supported by server")) if status == 0x81
|
709
|
-
mechanisms = content.split(' ')
|
710
|
-
raise NotImplementedError, "Dalli only supports the PLAIN authentication mechanism" if !mechanisms.include?('PLAIN')
|
711
|
-
|
712
|
-
# request
|
713
|
-
mechanism = 'PLAIN'
|
714
|
-
msg = "\x0#{username}\x0#{password}"
|
715
|
-
req = [REQUEST, OPCODES[:auth_request], mechanism.bytesize, 0, 0, 0, mechanism.bytesize + msg.bytesize, 0, 0, mechanism, msg].pack(FORMAT[:auth_request])
|
716
|
-
write(req)
|
717
|
-
|
718
|
-
(extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
|
719
|
-
raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
|
720
|
-
content = read(count)
|
721
|
-
return Dalli.logger.info("Dalli/SASL: #{content}") if status == 0
|
722
|
-
|
723
|
-
raise Dalli::DalliError, "Error authenticating: #{status}" unless status == 0x21
|
724
|
-
raise NotImplementedError, "No two-step authentication mechanisms supported"
|
725
|
-
# (step, msg) = sasl.receive('challenge', content)
|
726
|
-
# raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
|
727
|
-
end
|
728
|
-
|
729
|
-
def parse_hostname(str)
|
730
|
-
res = str.match(/\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/)
|
731
|
-
raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
|
732
|
-
hostnam = res[2] || res[1]
|
733
|
-
if hostnam =~ /\A\//
|
734
|
-
socket_type = :unix
|
735
|
-
# in case of unix socket, allow only setting of weight, not port
|
736
|
-
raise Dalli::DalliError, "Could not parse hostname #{str}" if res[4]
|
737
|
-
weigh = res[3]
|
738
|
-
else
|
739
|
-
socket_type = :tcp
|
740
|
-
por = res[3] || DEFAULT_PORT
|
741
|
-
por = Integer(por)
|
742
|
-
weigh = res[4]
|
743
|
-
end
|
744
|
-
weigh ||= DEFAULT_WEIGHT
|
745
|
-
weigh = Integer(weigh)
|
746
|
-
return hostnam, por, weigh, socket_type
|
747
|
-
end
|
748
|
-
end
|
3
|
+
module Dalli # rubocop:disable Style/Documentation
|
4
|
+
warn 'Dalli::Server is deprecated, use Dalli::Protocol::Binary instead'
|
5
|
+
Server = Protocol::Binary
|
749
6
|
end
|