dalli 2.7.8 → 3.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/{History.md → CHANGELOG.md} +168 -0
- data/Gemfile +5 -1
- data/README.md +27 -223
- data/lib/dalli/cas/client.rb +1 -57
- data/lib/dalli/client.rb +227 -254
- data/lib/dalli/compressor.rb +12 -2
- data/lib/dalli/key_manager.rb +121 -0
- data/lib/dalli/options.rb +6 -7
- data/lib/dalli/pipelined_getter.rb +177 -0
- data/lib/dalli/protocol/base.rb +241 -0
- data/lib/dalli/protocol/binary/request_formatter.rb +117 -0
- data/lib/dalli/protocol/binary/response_header.rb +36 -0
- data/lib/dalli/protocol/binary/response_processor.rb +239 -0
- data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
- data/lib/dalli/protocol/binary.rb +173 -0
- data/lib/dalli/protocol/connection_manager.rb +252 -0
- data/lib/dalli/protocol/meta/key_regularizer.rb +31 -0
- data/lib/dalli/protocol/meta/request_formatter.rb +121 -0
- data/lib/dalli/protocol/meta/response_processor.rb +211 -0
- data/lib/dalli/protocol/meta.rb +178 -0
- data/lib/dalli/protocol/response_buffer.rb +54 -0
- data/lib/dalli/protocol/server_config_parser.rb +86 -0
- data/lib/dalli/protocol/ttl_sanitizer.rb +45 -0
- data/lib/dalli/protocol/value_compressor.rb +85 -0
- data/lib/dalli/protocol/value_marshaller.rb +59 -0
- data/lib/dalli/protocol/value_serializer.rb +91 -0
- data/lib/dalli/protocol.rb +8 -0
- data/lib/dalli/ring.rb +94 -83
- data/lib/dalli/server.rb +3 -746
- data/lib/dalli/servers_arg_normalizer.rb +54 -0
- data/lib/dalli/socket.rb +117 -137
- data/lib/dalli/version.rb +4 -1
- data/lib/dalli.rb +43 -15
- data/lib/rack/session/dalli.rb +103 -94
- metadata +65 -28
- data/lib/action_dispatch/middleware/session/dalli_store.rb +0 -82
- data/lib/active_support/cache/dalli_store.rb +0 -429
- data/lib/dalli/railtie.rb +0 -8
data/lib/dalli/server.rb
CHANGED
@@ -1,749 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'socket'
|
3
|
-
require 'timeout'
|
4
2
|
|
5
|
-
module Dalli
|
6
|
-
|
7
|
-
|
8
|
-
attr_accessor :port
|
9
|
-
attr_accessor :weight
|
10
|
-
attr_accessor :options
|
11
|
-
attr_reader :sock
|
12
|
-
attr_reader :socket_type # possible values: :unix, :tcp
|
13
|
-
|
14
|
-
DEFAULT_PORT = 11211
|
15
|
-
DEFAULT_WEIGHT = 1
|
16
|
-
DEFAULTS = {
|
17
|
-
# seconds between trying to contact a remote server
|
18
|
-
:down_retry_delay => 60,
|
19
|
-
# connect/read/write timeout for socket operations
|
20
|
-
:socket_timeout => 0.5,
|
21
|
-
# times a socket operation may fail before considering the server dead
|
22
|
-
:socket_max_failures => 2,
|
23
|
-
# amount of time to sleep between retries when a failure occurs
|
24
|
-
:socket_failure_delay => 0.01,
|
25
|
-
# max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
|
26
|
-
:value_max_bytes => 1024 * 1024,
|
27
|
-
# surpassing value_max_bytes either warns (false) or throws (true)
|
28
|
-
:error_when_over_max_size => false,
|
29
|
-
:compressor => Compressor,
|
30
|
-
# min byte size to attempt compression
|
31
|
-
:compression_min_size => 1024,
|
32
|
-
# max byte size for compression
|
33
|
-
:compression_max_size => false,
|
34
|
-
:serializer => Marshal,
|
35
|
-
:username => nil,
|
36
|
-
:password => nil,
|
37
|
-
:keepalive => true,
|
38
|
-
# max byte size for SO_SNDBUF
|
39
|
-
:sndbuf => nil,
|
40
|
-
# max byte size for SO_RCVBUF
|
41
|
-
:rcvbuf => nil
|
42
|
-
}
|
43
|
-
|
44
|
-
def initialize(attribs, options = {})
|
45
|
-
@hostname, @port, @weight, @socket_type = parse_hostname(attribs)
|
46
|
-
@fail_count = 0
|
47
|
-
@down_at = nil
|
48
|
-
@last_down_at = nil
|
49
|
-
@options = DEFAULTS.merge(options)
|
50
|
-
@sock = nil
|
51
|
-
@msg = nil
|
52
|
-
@error = nil
|
53
|
-
@pid = nil
|
54
|
-
@inprogress = nil
|
55
|
-
end
|
56
|
-
|
57
|
-
def name
|
58
|
-
if socket_type == :unix
|
59
|
-
hostname
|
60
|
-
else
|
61
|
-
"#{hostname}:#{port}"
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
# Chokepoint method for instrumentation
|
66
|
-
def request(op, *args)
|
67
|
-
verify_state
|
68
|
-
raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}. If you are sure it is running, ensure memcached version is > 1.4." unless alive?
|
69
|
-
begin
|
70
|
-
send(op, *args)
|
71
|
-
rescue Dalli::MarshalError => ex
|
72
|
-
Dalli.logger.error "Marshalling error for key '#{args.first}': #{ex.message}"
|
73
|
-
Dalli.logger.error "You are trying to cache a Ruby object which cannot be serialized to memcached."
|
74
|
-
Dalli.logger.error ex.backtrace.join("\n\t")
|
75
|
-
false
|
76
|
-
rescue Dalli::DalliError, Dalli::NetworkError, Dalli::ValueOverMaxSize, Timeout::Error
|
77
|
-
raise
|
78
|
-
rescue => ex
|
79
|
-
Dalli.logger.error "Unexpected exception during Dalli request: #{ex.class.name}: #{ex.message}"
|
80
|
-
Dalli.logger.error ex.backtrace.join("\n\t")
|
81
|
-
down!
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
def alive?
|
86
|
-
return true if @sock
|
87
|
-
|
88
|
-
if @last_down_at && @last_down_at + options[:down_retry_delay] >= Time.now
|
89
|
-
time = @last_down_at + options[:down_retry_delay] - Time.now
|
90
|
-
Dalli.logger.debug { "down_retry_delay not reached for #{name} (%.3f seconds left)" % time }
|
91
|
-
return false
|
92
|
-
end
|
93
|
-
|
94
|
-
connect
|
95
|
-
!!@sock
|
96
|
-
rescue Dalli::NetworkError
|
97
|
-
false
|
98
|
-
end
|
99
|
-
|
100
|
-
def close
|
101
|
-
return unless @sock
|
102
|
-
@sock.close rescue nil
|
103
|
-
@sock = nil
|
104
|
-
@pid = nil
|
105
|
-
@inprogress = false
|
106
|
-
end
|
107
|
-
|
108
|
-
def lock!
|
109
|
-
end
|
110
|
-
|
111
|
-
def unlock!
|
112
|
-
end
|
113
|
-
|
114
|
-
def serializer
|
115
|
-
@options[:serializer]
|
116
|
-
end
|
117
|
-
|
118
|
-
def compressor
|
119
|
-
@options[:compressor]
|
120
|
-
end
|
121
|
-
|
122
|
-
# Start reading key/value pairs from this connection. This is usually called
|
123
|
-
# after a series of GETKQ commands. A NOOP is sent, and the server begins
|
124
|
-
# flushing responses for kv pairs that were found.
|
125
|
-
#
|
126
|
-
# Returns nothing.
|
127
|
-
def multi_response_start
|
128
|
-
verify_state
|
129
|
-
write_noop
|
130
|
-
@multi_buffer = String.new('')
|
131
|
-
@position = 0
|
132
|
-
@inprogress = true
|
133
|
-
end
|
134
|
-
|
135
|
-
# Did the last call to #multi_response_start complete successfully?
|
136
|
-
def multi_response_completed?
|
137
|
-
@multi_buffer.nil?
|
138
|
-
end
|
139
|
-
|
140
|
-
# Attempt to receive and parse as many key/value pairs as possible
|
141
|
-
# from this server. After #multi_response_start, this should be invoked
|
142
|
-
# repeatedly whenever this server's socket is readable until
|
143
|
-
# #multi_response_completed?.
|
144
|
-
#
|
145
|
-
# Returns a Hash of kv pairs received.
|
146
|
-
def multi_response_nonblock
|
147
|
-
raise 'multi_response has completed' if @multi_buffer.nil?
|
148
|
-
|
149
|
-
@multi_buffer << @sock.read_available
|
150
|
-
buf = @multi_buffer
|
151
|
-
pos = @position
|
152
|
-
values = {}
|
153
|
-
|
154
|
-
while buf.bytesize - pos >= 24
|
155
|
-
header = buf.slice(pos, 24)
|
156
|
-
(key_length, _, body_length, cas) = header.unpack(KV_HEADER)
|
157
|
-
|
158
|
-
if key_length == 0
|
159
|
-
# all done!
|
160
|
-
@multi_buffer = nil
|
161
|
-
@position = nil
|
162
|
-
@inprogress = false
|
163
|
-
break
|
164
|
-
|
165
|
-
elsif buf.bytesize - pos >= 24 + body_length
|
166
|
-
flags = buf.slice(pos + 24, 4).unpack('N')[0]
|
167
|
-
key = buf.slice(pos + 24 + 4, key_length)
|
168
|
-
value = buf.slice(pos + 24 + 4 + key_length, body_length - key_length - 4) if body_length - key_length - 4 > 0
|
169
|
-
|
170
|
-
pos = pos + 24 + body_length
|
171
|
-
|
172
|
-
begin
|
173
|
-
values[key] = [deserialize(value, flags), cas]
|
174
|
-
rescue DalliError
|
175
|
-
end
|
176
|
-
|
177
|
-
else
|
178
|
-
# not enough data yet, wait for more
|
179
|
-
break
|
180
|
-
end
|
181
|
-
end
|
182
|
-
@position = pos
|
183
|
-
|
184
|
-
values
|
185
|
-
rescue SystemCallError, Timeout::Error, EOFError => e
|
186
|
-
failure!(e)
|
187
|
-
end
|
188
|
-
|
189
|
-
# Abort an earlier #multi_response_start. Used to signal an external
|
190
|
-
# timeout. The underlying socket is disconnected, and the exception is
|
191
|
-
# swallowed.
|
192
|
-
#
|
193
|
-
# Returns nothing.
|
194
|
-
def multi_response_abort
|
195
|
-
@multi_buffer = nil
|
196
|
-
@position = nil
|
197
|
-
@inprogress = false
|
198
|
-
failure!(RuntimeError.new('External timeout'))
|
199
|
-
rescue NetworkError
|
200
|
-
true
|
201
|
-
end
|
202
|
-
|
203
|
-
# NOTE: Additional public methods should be overridden in Dalli::Threadsafe
|
204
|
-
|
205
|
-
private
|
206
|
-
|
207
|
-
def verify_state
|
208
|
-
failure!(RuntimeError.new('Already writing to socket')) if @inprogress
|
209
|
-
if @pid && @pid != Process.pid
|
210
|
-
message = 'Fork detected, re-connecting child process...'
|
211
|
-
Dalli.logger.info { message }
|
212
|
-
reconnect! message
|
213
|
-
end
|
214
|
-
end
|
215
|
-
|
216
|
-
def reconnect!(message)
|
217
|
-
close
|
218
|
-
sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
|
219
|
-
raise Dalli::NetworkError, message
|
220
|
-
end
|
221
|
-
|
222
|
-
def failure!(exception)
|
223
|
-
message = "#{name} failed (count: #{@fail_count}) #{exception.class}: #{exception.message}"
|
224
|
-
Dalli.logger.warn { message }
|
225
|
-
|
226
|
-
@fail_count += 1
|
227
|
-
if @fail_count >= options[:socket_max_failures]
|
228
|
-
down!
|
229
|
-
else
|
230
|
-
reconnect! 'Socket operation failed, retrying...'
|
231
|
-
end
|
232
|
-
end
|
233
|
-
|
234
|
-
def down!
|
235
|
-
close
|
236
|
-
|
237
|
-
@last_down_at = Time.now
|
238
|
-
|
239
|
-
if @down_at
|
240
|
-
time = Time.now - @down_at
|
241
|
-
Dalli.logger.debug { "#{name} is still down (for %.3f seconds now)" % time }
|
242
|
-
else
|
243
|
-
@down_at = @last_down_at
|
244
|
-
Dalli.logger.warn { "#{name} is down" }
|
245
|
-
end
|
246
|
-
|
247
|
-
@error = $! && $!.class.name
|
248
|
-
@msg = @msg || ($! && $!.message && !$!.message.empty? && $!.message)
|
249
|
-
raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
|
250
|
-
end
|
251
|
-
|
252
|
-
def up!
|
253
|
-
if @down_at
|
254
|
-
time = Time.now - @down_at
|
255
|
-
Dalli.logger.warn { "#{name} is back (downtime was %.3f seconds)" % time }
|
256
|
-
end
|
257
|
-
|
258
|
-
@fail_count = 0
|
259
|
-
@down_at = nil
|
260
|
-
@last_down_at = nil
|
261
|
-
@msg = nil
|
262
|
-
@error = nil
|
263
|
-
end
|
264
|
-
|
265
|
-
def multi?
|
266
|
-
Thread.current[:dalli_multi]
|
267
|
-
end
|
268
|
-
|
269
|
-
def get(key, options=nil)
|
270
|
-
req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
|
271
|
-
write(req)
|
272
|
-
generic_response(true, !!(options && options.is_a?(Hash) && options[:cache_nils]))
|
273
|
-
end
|
274
|
-
|
275
|
-
def send_multiget(keys)
|
276
|
-
req = String.new("")
|
277
|
-
keys.each do |key|
|
278
|
-
req << [REQUEST, OPCODES[:getkq], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:getkq])
|
279
|
-
end
|
280
|
-
# Could send noop here instead of in multi_response_start
|
281
|
-
write(req)
|
282
|
-
end
|
283
|
-
|
284
|
-
def set(key, value, ttl, cas, options)
|
285
|
-
(value, flags) = serialize(key, value, options)
|
286
|
-
ttl = sanitize_ttl(ttl)
|
287
|
-
|
288
|
-
guard_max_value(key, value) do
|
289
|
-
req = [REQUEST, OPCODES[multi? ? :setq : :set], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:set])
|
290
|
-
write(req)
|
291
|
-
cas_response unless multi?
|
292
|
-
end
|
293
|
-
end
|
294
|
-
|
295
|
-
def add(key, value, ttl, options)
|
296
|
-
(value, flags) = serialize(key, value, options)
|
297
|
-
ttl = sanitize_ttl(ttl)
|
298
|
-
|
299
|
-
guard_max_value(key, value) do
|
300
|
-
req = [REQUEST, OPCODES[multi? ? :addq : :add], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:add])
|
301
|
-
write(req)
|
302
|
-
cas_response unless multi?
|
303
|
-
end
|
304
|
-
end
|
305
|
-
|
306
|
-
def replace(key, value, ttl, cas, options)
|
307
|
-
(value, flags) = serialize(key, value, options)
|
308
|
-
ttl = sanitize_ttl(ttl)
|
309
|
-
|
310
|
-
guard_max_value(key, value) do
|
311
|
-
req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:replace])
|
312
|
-
write(req)
|
313
|
-
cas_response unless multi?
|
314
|
-
end
|
315
|
-
end
|
316
|
-
|
317
|
-
def delete(key, cas)
|
318
|
-
req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, cas, key].pack(FORMAT[:delete])
|
319
|
-
write(req)
|
320
|
-
generic_response unless multi?
|
321
|
-
end
|
322
|
-
|
323
|
-
def flush(ttl)
|
324
|
-
req = [REQUEST, OPCODES[:flush], 0, 4, 0, 0, 4, 0, 0, 0].pack(FORMAT[:flush])
|
325
|
-
write(req)
|
326
|
-
generic_response
|
327
|
-
end
|
328
|
-
|
329
|
-
def decr_incr(opcode, key, count, ttl, default)
|
330
|
-
expiry = default ? sanitize_ttl(ttl) : 0xFFFFFFFF
|
331
|
-
default ||= 0
|
332
|
-
(h, l) = split(count)
|
333
|
-
(dh, dl) = split(default)
|
334
|
-
req = [REQUEST, OPCODES[opcode], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[opcode])
|
335
|
-
write(req)
|
336
|
-
body = generic_response
|
337
|
-
body ? body.unpack('Q>').first : body
|
338
|
-
end
|
339
|
-
|
340
|
-
def decr(key, count, ttl, default)
|
341
|
-
decr_incr :decr, key, count, ttl, default
|
342
|
-
end
|
343
|
-
|
344
|
-
def incr(key, count, ttl, default)
|
345
|
-
decr_incr :incr, key, count, ttl, default
|
346
|
-
end
|
347
|
-
|
348
|
-
def write_append_prepend(opcode, key, value)
|
349
|
-
write_generic [REQUEST, OPCODES[opcode], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[opcode])
|
350
|
-
end
|
351
|
-
|
352
|
-
def write_generic(bytes)
|
353
|
-
write(bytes)
|
354
|
-
generic_response
|
355
|
-
end
|
356
|
-
|
357
|
-
def write_noop
|
358
|
-
req = [REQUEST, OPCODES[:noop], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
|
359
|
-
write(req)
|
360
|
-
end
|
361
|
-
|
362
|
-
# Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
|
363
|
-
# We need to read all the responses at once.
|
364
|
-
def noop
|
365
|
-
write_noop
|
366
|
-
multi_response
|
367
|
-
end
|
368
|
-
|
369
|
-
def append(key, value)
|
370
|
-
write_append_prepend :append, key, value
|
371
|
-
end
|
372
|
-
|
373
|
-
def prepend(key, value)
|
374
|
-
write_append_prepend :prepend, key, value
|
375
|
-
end
|
376
|
-
|
377
|
-
def stats(info='')
|
378
|
-
req = [REQUEST, OPCODES[:stat], info.bytesize, 0, 0, 0, info.bytesize, 0, 0, info].pack(FORMAT[:stat])
|
379
|
-
write(req)
|
380
|
-
keyvalue_response
|
381
|
-
end
|
382
|
-
|
383
|
-
def reset_stats
|
384
|
-
write_generic [REQUEST, OPCODES[:stat], 'reset'.bytesize, 0, 0, 0, 'reset'.bytesize, 0, 0, 'reset'].pack(FORMAT[:stat])
|
385
|
-
end
|
386
|
-
|
387
|
-
def cas(key)
|
388
|
-
req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
|
389
|
-
write(req)
|
390
|
-
data_cas_response
|
391
|
-
end
|
392
|
-
|
393
|
-
def version
|
394
|
-
write_generic [REQUEST, OPCODES[:version], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
|
395
|
-
end
|
396
|
-
|
397
|
-
def touch(key, ttl)
|
398
|
-
ttl = sanitize_ttl(ttl)
|
399
|
-
write_generic [REQUEST, OPCODES[:touch], key.bytesize, 4, 0, 0, key.bytesize + 4, 0, 0, ttl, key].pack(FORMAT[:touch])
|
400
|
-
end
|
401
|
-
|
402
|
-
# http://www.hjp.at/zettel/m/memcached_flags.rxml
|
403
|
-
# Looks like most clients use bit 0 to indicate native language serialization
|
404
|
-
# and bit 1 to indicate gzip compression.
|
405
|
-
FLAG_SERIALIZED = 0x1
|
406
|
-
FLAG_COMPRESSED = 0x2
|
407
|
-
|
408
|
-
def serialize(key, value, options=nil)
|
409
|
-
marshalled = false
|
410
|
-
value = unless options && options[:raw]
|
411
|
-
marshalled = true
|
412
|
-
begin
|
413
|
-
self.serializer.dump(value)
|
414
|
-
rescue Timeout::Error => e
|
415
|
-
raise e
|
416
|
-
rescue => ex
|
417
|
-
# Marshalling can throw several different types of generic Ruby exceptions.
|
418
|
-
# Convert to a specific exception so we can special case it higher up the stack.
|
419
|
-
exc = Dalli::MarshalError.new(ex.message)
|
420
|
-
exc.set_backtrace ex.backtrace
|
421
|
-
raise exc
|
422
|
-
end
|
423
|
-
else
|
424
|
-
value.to_s
|
425
|
-
end
|
426
|
-
compressed = false
|
427
|
-
set_compress_option = true if options && options[:compress]
|
428
|
-
if (@options[:compress] || set_compress_option) && value.bytesize >= @options[:compression_min_size] &&
|
429
|
-
(!@options[:compression_max_size] || value.bytesize <= @options[:compression_max_size])
|
430
|
-
value = self.compressor.compress(value)
|
431
|
-
compressed = true
|
432
|
-
end
|
433
|
-
|
434
|
-
flags = 0
|
435
|
-
flags |= FLAG_COMPRESSED if compressed
|
436
|
-
flags |= FLAG_SERIALIZED if marshalled
|
437
|
-
[value, flags]
|
438
|
-
end
|
439
|
-
|
440
|
-
def deserialize(value, flags)
|
441
|
-
value = self.compressor.decompress(value) if (flags & FLAG_COMPRESSED) != 0
|
442
|
-
value = self.serializer.load(value) if (flags & FLAG_SERIALIZED) != 0
|
443
|
-
value
|
444
|
-
rescue TypeError
|
445
|
-
raise if $!.message !~ /needs to have method `_load'|exception class\/object expected|instance of IO needed|incompatible marshal file format/
|
446
|
-
raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
|
447
|
-
rescue ArgumentError
|
448
|
-
raise if $!.message !~ /undefined class|marshal data too short/
|
449
|
-
raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
|
450
|
-
rescue Zlib::Error
|
451
|
-
raise UnmarshalError, "Unable to uncompress value: #{$!.message}"
|
452
|
-
end
|
453
|
-
|
454
|
-
def data_cas_response
|
455
|
-
(extras, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
|
456
|
-
data = read(count) if count > 0
|
457
|
-
if status == 1
|
458
|
-
nil
|
459
|
-
elsif status != 0
|
460
|
-
raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
|
461
|
-
elsif data
|
462
|
-
flags = data[0...extras].unpack('N')[0]
|
463
|
-
value = data[extras..-1]
|
464
|
-
data = deserialize(value, flags)
|
465
|
-
end
|
466
|
-
[data, cas]
|
467
|
-
end
|
468
|
-
|
469
|
-
CAS_HEADER = '@4CCnNNQ'
|
470
|
-
NORMAL_HEADER = '@4CCnN'
|
471
|
-
KV_HEADER = '@2n@6nN@16Q'
|
472
|
-
|
473
|
-
def guard_max_value(key, value)
|
474
|
-
if value.bytesize <= @options[:value_max_bytes]
|
475
|
-
yield
|
476
|
-
else
|
477
|
-
message = "Value for #{key} over max size: #{@options[:value_max_bytes]} <= #{value.bytesize}"
|
478
|
-
raise Dalli::ValueOverMaxSize, message if @options[:error_when_over_max_size]
|
479
|
-
|
480
|
-
Dalli.logger.warn message
|
481
|
-
false
|
482
|
-
end
|
483
|
-
end
|
484
|
-
|
485
|
-
# https://github.com/memcached/memcached/blob/master/doc/protocol.txt#L79
|
486
|
-
# > An expiration time, in seconds. Can be up to 30 days. After 30 days, is treated as a unix timestamp of an exact date.
|
487
|
-
MAX_ACCEPTABLE_EXPIRATION_INTERVAL = 30*24*60*60 # 30 days
|
488
|
-
def sanitize_ttl(ttl)
|
489
|
-
ttl_as_i = ttl.to_i
|
490
|
-
return ttl_as_i if ttl_as_i <= MAX_ACCEPTABLE_EXPIRATION_INTERVAL
|
491
|
-
now = Time.now.to_i
|
492
|
-
return ttl_as_i if ttl_as_i > now # already a timestamp
|
493
|
-
Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached, converting to an expiration timestamp"
|
494
|
-
now + ttl_as_i
|
495
|
-
end
|
496
|
-
|
497
|
-
# Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
|
498
|
-
class NilObject; end
|
499
|
-
NOT_FOUND = NilObject.new
|
500
|
-
|
501
|
-
def generic_response(unpack=false, cache_nils=false)
|
502
|
-
(extras, _, status, count) = read_header.unpack(NORMAL_HEADER)
|
503
|
-
data = read(count) if count > 0
|
504
|
-
if status == 1
|
505
|
-
cache_nils ? NOT_FOUND : nil
|
506
|
-
elsif status == 2 || status == 5
|
507
|
-
false # Not stored, normal status for add operation
|
508
|
-
elsif status != 0
|
509
|
-
raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
|
510
|
-
elsif data
|
511
|
-
flags = data[0...extras].unpack('N')[0]
|
512
|
-
value = data[extras..-1]
|
513
|
-
unpack ? deserialize(value, flags) : value
|
514
|
-
else
|
515
|
-
true
|
516
|
-
end
|
517
|
-
end
|
518
|
-
|
519
|
-
def cas_response
|
520
|
-
(_, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
|
521
|
-
read(count) if count > 0 # this is potential data that we don't care about
|
522
|
-
if status == 1
|
523
|
-
nil
|
524
|
-
elsif status == 2 || status == 5
|
525
|
-
false # Not stored, normal status for add operation
|
526
|
-
elsif status != 0
|
527
|
-
raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
|
528
|
-
else
|
529
|
-
cas
|
530
|
-
end
|
531
|
-
end
|
532
|
-
|
533
|
-
def keyvalue_response
|
534
|
-
hash = {}
|
535
|
-
while true
|
536
|
-
(key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
|
537
|
-
return hash if key_length == 0
|
538
|
-
key = read(key_length)
|
539
|
-
value = read(body_length - key_length) if body_length - key_length > 0
|
540
|
-
hash[key] = value
|
541
|
-
end
|
542
|
-
end
|
543
|
-
|
544
|
-
def multi_response
|
545
|
-
hash = {}
|
546
|
-
while true
|
547
|
-
(key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
|
548
|
-
return hash if key_length == 0
|
549
|
-
flags = read(4).unpack('N')[0]
|
550
|
-
key = read(key_length)
|
551
|
-
value = read(body_length - key_length - 4) if body_length - key_length - 4 > 0
|
552
|
-
hash[key] = deserialize(value, flags)
|
553
|
-
end
|
554
|
-
end
|
555
|
-
|
556
|
-
def write(bytes)
|
557
|
-
begin
|
558
|
-
@inprogress = true
|
559
|
-
result = @sock.write(bytes)
|
560
|
-
@inprogress = false
|
561
|
-
result
|
562
|
-
rescue SystemCallError, Timeout::Error => e
|
563
|
-
failure!(e)
|
564
|
-
end
|
565
|
-
end
|
566
|
-
|
567
|
-
def read(count)
|
568
|
-
begin
|
569
|
-
@inprogress = true
|
570
|
-
data = @sock.readfull(count)
|
571
|
-
@inprogress = false
|
572
|
-
data
|
573
|
-
rescue SystemCallError, Timeout::Error, EOFError => e
|
574
|
-
failure!(e)
|
575
|
-
end
|
576
|
-
end
|
577
|
-
|
578
|
-
def read_header
|
579
|
-
read(24) || raise(Dalli::NetworkError, 'No response')
|
580
|
-
end
|
581
|
-
|
582
|
-
def connect
|
583
|
-
Dalli.logger.debug { "Dalli::Server#connect #{name}" }
|
584
|
-
|
585
|
-
begin
|
586
|
-
@pid = Process.pid
|
587
|
-
if socket_type == :unix
|
588
|
-
@sock = KSocket::UNIX.open(hostname, self, options)
|
589
|
-
else
|
590
|
-
@sock = KSocket::TCP.open(hostname, port, self, options)
|
591
|
-
end
|
592
|
-
sasl_authentication if need_auth?
|
593
|
-
@version = version # trigger actual connect
|
594
|
-
up!
|
595
|
-
rescue Dalli::DalliError # SASL auth failure
|
596
|
-
raise
|
597
|
-
rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
|
598
|
-
# SocketError = DNS resolution failure
|
599
|
-
failure!(e)
|
600
|
-
end
|
601
|
-
end
|
602
|
-
|
603
|
-
def split(n)
|
604
|
-
[n >> 32, 0xFFFFFFFF & n]
|
605
|
-
end
|
606
|
-
|
607
|
-
REQUEST = 0x80
|
608
|
-
RESPONSE = 0x81
|
609
|
-
|
610
|
-
# Response codes taken from:
|
611
|
-
# https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
|
612
|
-
RESPONSE_CODES = {
|
613
|
-
0 => 'No error',
|
614
|
-
1 => 'Key not found',
|
615
|
-
2 => 'Key exists',
|
616
|
-
3 => 'Value too large',
|
617
|
-
4 => 'Invalid arguments',
|
618
|
-
5 => 'Item not stored',
|
619
|
-
6 => 'Incr/decr on a non-numeric value',
|
620
|
-
7 => 'The vbucket belongs to another server',
|
621
|
-
8 => 'Authentication error',
|
622
|
-
9 => 'Authentication continue',
|
623
|
-
0x20 => 'Authentication required',
|
624
|
-
0x81 => 'Unknown command',
|
625
|
-
0x82 => 'Out of memory',
|
626
|
-
0x83 => 'Not supported',
|
627
|
-
0x84 => 'Internal error',
|
628
|
-
0x85 => 'Busy',
|
629
|
-
0x86 => 'Temporary failure'
|
630
|
-
}
|
631
|
-
|
632
|
-
OPCODES = {
|
633
|
-
:get => 0x00,
|
634
|
-
:set => 0x01,
|
635
|
-
:add => 0x02,
|
636
|
-
:replace => 0x03,
|
637
|
-
:delete => 0x04,
|
638
|
-
:incr => 0x05,
|
639
|
-
:decr => 0x06,
|
640
|
-
:flush => 0x08,
|
641
|
-
:noop => 0x0A,
|
642
|
-
:version => 0x0B,
|
643
|
-
:getkq => 0x0D,
|
644
|
-
:append => 0x0E,
|
645
|
-
:prepend => 0x0F,
|
646
|
-
:stat => 0x10,
|
647
|
-
:setq => 0x11,
|
648
|
-
:addq => 0x12,
|
649
|
-
:replaceq => 0x13,
|
650
|
-
:deleteq => 0x14,
|
651
|
-
:incrq => 0x15,
|
652
|
-
:decrq => 0x16,
|
653
|
-
:auth_negotiation => 0x20,
|
654
|
-
:auth_request => 0x21,
|
655
|
-
:auth_continue => 0x22,
|
656
|
-
:touch => 0x1C,
|
657
|
-
}
|
658
|
-
|
659
|
-
HEADER = "CCnCCnNNQ"
|
660
|
-
OP_FORMAT = {
|
661
|
-
:get => 'a*',
|
662
|
-
:set => 'NNa*a*',
|
663
|
-
:add => 'NNa*a*',
|
664
|
-
:replace => 'NNa*a*',
|
665
|
-
:delete => 'a*',
|
666
|
-
:incr => 'NNNNNa*',
|
667
|
-
:decr => 'NNNNNa*',
|
668
|
-
:flush => 'N',
|
669
|
-
:noop => '',
|
670
|
-
:getkq => 'a*',
|
671
|
-
:version => '',
|
672
|
-
:stat => 'a*',
|
673
|
-
:append => 'a*a*',
|
674
|
-
:prepend => 'a*a*',
|
675
|
-
:auth_request => 'a*a*',
|
676
|
-
:auth_continue => 'a*a*',
|
677
|
-
:touch => 'Na*',
|
678
|
-
}
|
679
|
-
FORMAT = OP_FORMAT.inject({}) { |memo, (k, v)| memo[k] = HEADER + v; memo }
|
680
|
-
|
681
|
-
|
682
|
-
#######
|
683
|
-
# SASL authentication support for NorthScale
|
684
|
-
#######
|
685
|
-
|
686
|
-
def need_auth?
|
687
|
-
@options[:username] || ENV['MEMCACHE_USERNAME']
|
688
|
-
end
|
689
|
-
|
690
|
-
def username
|
691
|
-
@options[:username] || ENV['MEMCACHE_USERNAME']
|
692
|
-
end
|
693
|
-
|
694
|
-
def password
|
695
|
-
@options[:password] || ENV['MEMCACHE_PASSWORD']
|
696
|
-
end
|
697
|
-
|
698
|
-
def sasl_authentication
|
699
|
-
Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
|
700
|
-
|
701
|
-
# negotiate
|
702
|
-
req = [REQUEST, OPCODES[:auth_negotiation], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
|
703
|
-
write(req)
|
704
|
-
|
705
|
-
(extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
|
706
|
-
raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
|
707
|
-
content = read(count).gsub(/\u0000/, ' ')
|
708
|
-
return (Dalli.logger.debug("Authentication not required/supported by server")) if status == 0x81
|
709
|
-
mechanisms = content.split(' ')
|
710
|
-
raise NotImplementedError, "Dalli only supports the PLAIN authentication mechanism" if !mechanisms.include?('PLAIN')
|
711
|
-
|
712
|
-
# request
|
713
|
-
mechanism = 'PLAIN'
|
714
|
-
msg = "\x0#{username}\x0#{password}"
|
715
|
-
req = [REQUEST, OPCODES[:auth_request], mechanism.bytesize, 0, 0, 0, mechanism.bytesize + msg.bytesize, 0, 0, mechanism, msg].pack(FORMAT[:auth_request])
|
716
|
-
write(req)
|
717
|
-
|
718
|
-
(extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
|
719
|
-
raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
|
720
|
-
content = read(count)
|
721
|
-
return Dalli.logger.info("Dalli/SASL: #{content}") if status == 0
|
722
|
-
|
723
|
-
raise Dalli::DalliError, "Error authenticating: #{status}" unless status == 0x21
|
724
|
-
raise NotImplementedError, "No two-step authentication mechanisms supported"
|
725
|
-
# (step, msg) = sasl.receive('challenge', content)
|
726
|
-
# raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
|
727
|
-
end
|
728
|
-
|
729
|
-
def parse_hostname(str)
|
730
|
-
res = str.match(/\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/)
|
731
|
-
raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
|
732
|
-
hostnam = res[2] || res[1]
|
733
|
-
if hostnam =~ /\A\//
|
734
|
-
socket_type = :unix
|
735
|
-
# in case of unix socket, allow only setting of weight, not port
|
736
|
-
raise Dalli::DalliError, "Could not parse hostname #{str}" if res[4]
|
737
|
-
weigh = res[3]
|
738
|
-
else
|
739
|
-
socket_type = :tcp
|
740
|
-
por = res[3] || DEFAULT_PORT
|
741
|
-
por = Integer(por)
|
742
|
-
weigh = res[4]
|
743
|
-
end
|
744
|
-
weigh ||= DEFAULT_WEIGHT
|
745
|
-
weigh = Integer(weigh)
|
746
|
-
return hostnam, por, weigh, socket_type
|
747
|
-
end
|
748
|
-
end
|
3
|
+
module Dalli # rubocop:disable Style/Documentation
|
4
|
+
warn 'Dalli::Server is deprecated, use Dalli::Protocol::Binary instead'
|
5
|
+
Server = Protocol::Binary
|
749
6
|
end
|