dalli 3.0.2 → 3.0.6

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of dalli might be problematic. Click here for more details.

@@ -1,20 +1,29 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "socket"
4
- require "timeout"
3
+ require 'English'
4
+ require 'forwardable'
5
+ require 'socket'
6
+ require 'timeout'
7
+
8
+ require_relative 'binary/request_formatter'
9
+ require_relative 'binary/response_processor'
10
+ require_relative 'binary/sasl_authentication'
5
11
 
6
12
  module Dalli
7
13
  module Protocol
14
+ ##
15
+ # Access point for a single Memcached server, accessed via Memcached's binary
16
+ # protocol. Contains logic for managing connection state to the server (retries, etc),
17
+ # formatting requests to the server, and unpacking responses.
18
+ ##
8
19
  class Binary
9
- attr_accessor :hostname
10
- attr_accessor :port
11
- attr_accessor :weight
12
- attr_accessor :options
13
- attr_reader :sock
14
- attr_reader :socket_type # possible values: :unix, :tcp
15
-
16
- DEFAULT_PORT = 11211
17
- DEFAULT_WEIGHT = 1
20
+ extend Forwardable
21
+
22
+ attr_accessor :hostname, :port, :weight, :options
23
+ attr_reader :sock, :socket_type
24
+
25
+ def_delegators :@value_marshaller, :serializer, :compressor, :compression_min_size, :compress_by_default?
26
+
18
27
  DEFAULTS = {
19
28
  # seconds between trying to contact a remote server
20
29
  down_retry_delay: 30,
@@ -24,33 +33,20 @@ module Dalli
24
33
  socket_max_failures: 2,
25
34
  # amount of time to sleep between retries when a failure occurs
26
35
  socket_failure_delay: 0.1,
27
- # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
28
- value_max_bytes: 1024 * 1024,
29
- compress: true,
30
- compressor: Compressor,
31
- # min byte size to attempt compression
32
- compression_min_size: 4 * 1024,
33
- serializer: Marshal,
34
36
  username: nil,
35
- password: nil,
36
- keepalive: true,
37
- # max byte size for SO_SNDBUF
38
- sndbuf: nil,
39
- # max byte size for SO_RCVBUF
40
- rcvbuf: nil
41
- }
37
+ password: nil
38
+ }.freeze
42
39
 
43
40
  def initialize(attribs, options = {})
44
- @hostname, @port, @weight, @socket_type = parse_hostname(attribs)
45
- @fail_count = 0
46
- @down_at = nil
47
- @last_down_at = nil
41
+ @hostname, @port, @weight, @socket_type, options = ServerConfigParser.parse(attribs, options)
48
42
  @options = DEFAULTS.merge(options)
43
+ @value_marshaller = ValueMarshaller.new(@options)
44
+ @response_processor = ResponseProcessor.new(self, @value_marshaller)
45
+
46
+ reset_down_info
49
47
  @sock = nil
50
- @msg = nil
51
- @error = nil
52
48
  @pid = nil
53
- @inprogress = nil
49
+ @request_in_progress = false
54
50
  end
55
51
 
56
52
  def name
@@ -61,33 +57,50 @@ module Dalli
61
57
  end
62
58
  end
63
59
 
64
- # Chokepoint method for instrumentation
65
- def request(op, *args)
60
+ # Chokepoint method for error handling and ensuring liveness
61
+ def request(opcode, *args)
66
62
  verify_state
67
- raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}. If you are sure it is running, ensure memcached version is > 1.4." unless alive?
63
+ # The alive? call has the side effect of connecting the underlying
64
+ # socket if it is not connected, or there's been a disconnect
65
+ # because of timeout or other error. Method raises an error
66
+ # if it can't connect
67
+ raise_memcached_down_err unless alive?
68
+
68
69
  begin
69
- send(op, *args)
70
- rescue Dalli::MarshalError => ex
71
- Dalli.logger.error "Marshalling error for key '#{args.first}': #{ex.message}"
72
- Dalli.logger.error "You are trying to cache a Ruby object which cannot be serialized to memcached."
70
+ send(opcode, *args)
71
+ rescue Dalli::MarshalError => e
72
+ log_marshall_err(args.first, e)
73
73
  raise
74
74
  rescue Dalli::DalliError, Dalli::NetworkError, Dalli::ValueOverMaxSize, Timeout::Error
75
75
  raise
76
- rescue => ex
77
- Dalli.logger.error "Unexpected exception during Dalli request: #{ex.class.name}: #{ex.message}"
78
- Dalli.logger.error ex.backtrace.join("\n\t")
76
+ rescue StandardError => e
77
+ log_unexpected_err(e)
79
78
  down!
80
79
  end
81
80
  end
82
81
 
82
+ def raise_memcached_down_err
83
+ raise Dalli::NetworkError,
84
+ "#{name} is down: #{@error} #{@msg}. If you are sure it is running, "\
85
+ "ensure memcached version is > #{::Dalli::MIN_SUPPORTED_MEMCACHED_VERSION}."
86
+ end
87
+
88
+ def log_marshall_err(key, err)
89
+ Dalli.logger.error "Marshalling error for key '#{key}': #{err.message}"
90
+ Dalli.logger.error 'You are trying to cache a Ruby object which cannot be serialized to memcached.'
91
+ end
92
+
93
+ def log_unexpected_err(err)
94
+ Dalli.logger.error "Unexpected exception during Dalli request: #{err.class.name}: #{err.message}"
95
+ Dalli.logger.error err.backtrace.join("\n\t")
96
+ end
97
+
98
+ # The socket connection to the underlying server is initialized as a side
99
+ # effect of this call. In fact, this is the ONLY place where that
100
+ # socket connection is initialized.
83
101
  def alive?
84
102
  return true if @sock
85
-
86
- if @last_down_at && @last_down_at + options[:down_retry_delay] >= Time.now
87
- time = @last_down_at + options[:down_retry_delay] - Time.now
88
- Dalli.logger.debug { "down_retry_delay not reached for #{name} (%.3f seconds left)" % time }
89
- return false
90
- end
103
+ return false unless reconnect_down_server?
91
104
 
92
105
  connect
93
106
  !!@sock
@@ -95,31 +108,37 @@ module Dalli
95
108
  false
96
109
  end
97
110
 
111
+ def reconnect_down_server?
112
+ return true unless @last_down_at
113
+
114
+ time_to_next_reconnect = @last_down_at + options[:down_retry_delay] - Time.now
115
+ return true unless time_to_next_reconnect.positive?
116
+
117
+ Dalli.logger.debug do
118
+ format('down_retry_delay not reached for %<name>s (%<time>.3f seconds left)', name: name,
119
+ time: time_to_next_reconnect)
120
+ end
121
+ false
122
+ end
123
+
124
+ # Closes the underlying socket and cleans up
125
+ # socket state.
98
126
  def close
99
127
  return unless @sock
128
+
100
129
  begin
101
130
  @sock.close
102
- rescue
131
+ rescue StandardError
103
132
  nil
104
133
  end
105
134
  @sock = nil
106
135
  @pid = nil
107
- @inprogress = false
108
- end
109
-
110
- def lock!
136
+ abort_request!
111
137
  end
112
138
 
113
- def unlock!
114
- end
139
+ def lock!; end
115
140
 
116
- def serializer
117
- @options[:serializer]
118
- end
119
-
120
- def compressor
121
- @options[:compressor]
122
- end
141
+ def unlock!; end
123
142
 
124
143
  # Start reading key/value pairs from this connection. This is usually called
125
144
  # after a series of GETKQ commands. A NOOP is sent, and the server begins
@@ -129,9 +148,9 @@ module Dalli
129
148
  def multi_response_start
130
149
  verify_state
131
150
  write_noop
132
- @multi_buffer = +""
151
+ @multi_buffer = +''
133
152
  @position = 0
134
- @inprogress = true
153
+ start_request!
135
154
  end
136
155
 
137
156
  # Did the last call to #multi_response_start complete successfully?
@@ -146,41 +165,39 @@ module Dalli
146
165
  #
147
166
  # Returns a Hash of kv pairs received.
148
167
  def multi_response_nonblock
149
- reconnect! "multi_response has completed" if @multi_buffer.nil?
168
+ reconnect! 'multi_response has completed' if @multi_buffer.nil?
150
169
 
151
170
  @multi_buffer << @sock.read_available
152
171
  buf = @multi_buffer
153
172
  pos = @position
154
173
  values = {}
155
174
 
156
- while buf.bytesize - pos >= 24
157
- header = buf.slice(pos, 24)
158
- (key_length, _, body_length, cas) = header.unpack(KV_HEADER)
175
+ while buf.bytesize - pos >= ResponseProcessor::RESP_HEADER_SIZE
176
+ header = buf.slice(pos, ResponseProcessor::RESP_HEADER_SIZE)
177
+ _, extra_len, key_len, body_len, cas = @response_processor.unpack_header(header)
159
178
 
160
- if key_length == 0
161
- # all done!
162
- @multi_buffer = nil
163
- @position = nil
164
- @inprogress = false
179
+ # We've reached the noop at the end of the pipeline
180
+ if key_len.zero?
181
+ finish_multi_response
165
182
  break
183
+ end
166
184
 
167
- elsif buf.bytesize - pos >= 24 + body_length
168
- flags = buf.slice(pos + 24, 4).unpack1("N")
169
- key = buf.slice(pos + 24 + 4, key_length)
170
- value = buf.slice(pos + 24 + 4 + key_length, body_length - key_length - 4) if body_length - key_length - 4 > 0
171
-
172
- pos = pos + 24 + body_length
173
-
174
- begin
175
- values[key] = [deserialize(value, flags), cas]
176
- rescue DalliError
177
- end
185
+ # Break and read more unless we already have the entire response for this header
186
+ resp_size = ResponseProcessor::RESP_HEADER_SIZE + body_len
187
+ break unless buf.bytesize - pos >= resp_size
178
188
 
179
- else
180
- # not enough data yet, wait for more
181
- break
189
+ body = buf.slice(pos + ResponseProcessor::RESP_HEADER_SIZE, body_len)
190
+ begin
191
+ key, value = @response_processor.unpack_response_body(extra_len, key_len, body, true)
192
+ values[key] = [value, cas]
193
+ rescue DalliError
194
+ # TODO: Determine if we should be swallowing
195
+ # this error
182
196
  end
197
+
198
+ pos = pos + ResponseProcessor::RESP_HEADER_SIZE + body_len
183
199
  end
200
+ # TODO: We should be discarding the already processed buffer at this point
184
201
  @position = pos
185
202
 
186
203
  values
@@ -188,6 +205,12 @@ module Dalli
188
205
  failure!(e)
189
206
  end
190
207
 
208
+ def finish_multi_response
209
+ @multi_buffer = nil
210
+ @position = nil
211
+ finish_request!
212
+ end
213
+
191
214
  # Abort an earlier #multi_response_start. Used to signal an external
192
215
  # timeout. The underlying socket is disconnected, and the exception is
193
216
  # swallowed.
@@ -196,31 +219,81 @@ module Dalli
196
219
  def multi_response_abort
197
220
  @multi_buffer = nil
198
221
  @position = nil
199
- @inprogress = false
200
- failure!(RuntimeError.new("External timeout"))
222
+ abort_request!
223
+ return true unless @sock
224
+
225
+ failure!(RuntimeError.new('External timeout'))
201
226
  rescue NetworkError
202
227
  true
203
228
  end
204
229
 
230
+ def read(count)
231
+ start_request!
232
+ data = @sock.readfull(count)
233
+ finish_request!
234
+ data
235
+ rescue SystemCallError, Timeout::Error, EOFError => e
236
+ failure!(e)
237
+ end
238
+
239
+ def write(bytes)
240
+ start_request!
241
+ result = @sock.write(bytes)
242
+ finish_request!
243
+ result
244
+ rescue SystemCallError, Timeout::Error => e
245
+ failure!(e)
246
+ end
247
+
248
+ def socket_timeout
249
+ @socket_timeout ||= @options[:socket_timeout]
250
+ end
251
+
205
252
  # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
206
253
 
207
254
  private
208
255
 
256
+ def request_in_progress?
257
+ @request_in_progress
258
+ end
259
+
260
+ def start_request!
261
+ @request_in_progress = true
262
+ end
263
+
264
+ def finish_request!
265
+ @request_in_progress = false
266
+ end
267
+
268
+ def abort_request!
269
+ @request_in_progress = false
270
+ end
271
+
209
272
  def verify_state
210
- failure!(RuntimeError.new("Already writing to socket")) if @inprogress
211
- if @pid && @pid != Process.pid
212
- message = "Fork detected, re-connecting child process..."
213
- Dalli.logger.info { message }
214
- reconnect! message
215
- end
273
+ failure!(RuntimeError.new('Already writing to socket')) if request_in_progress?
274
+ reconnect_on_fork if fork_detected?
275
+ end
276
+
277
+ def fork_detected?
278
+ @pid && @pid != Process.pid
216
279
  end
217
280
 
281
+ def reconnect_on_fork
282
+ message = 'Fork detected, re-connecting child process...'
283
+ Dalli.logger.info { message }
284
+ reconnect! message
285
+ end
286
+
287
+ # Marks the server instance as needing reconnect. Raises a
288
+ # Dalli::NetworkError with the specified message. Calls close
289
+ # to clean up socket state
218
290
  def reconnect!(message)
219
291
  close
220
292
  sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
221
293
  raise Dalli::NetworkError, message
222
294
  end
223
295
 
296
+ # Raises Dalli::NetworkError
224
297
  def failure!(exception)
225
298
  message = "#{name} failed (count: #{@fail_count}) #{exception.class}: #{exception.message}"
226
299
  Dalli.logger.warn { message }
@@ -229,34 +302,47 @@ module Dalli
229
302
  if @fail_count >= options[:socket_max_failures]
230
303
  down!
231
304
  else
232
- reconnect! "Socket operation failed, retrying..."
305
+ reconnect! 'Socket operation failed, retrying...'
233
306
  end
234
307
  end
235
308
 
309
+ # Marks the server instance as down. Updates the down_at state
310
+ # and raises an Dalli::NetworkError that includes the underlying
311
+ # error in the message. Calls close to clean up socket state
236
312
  def down!
237
313
  close
314
+ log_down_detected
315
+
316
+ @error = $ERROR_INFO&.class&.name
317
+ @msg ||= $ERROR_INFO&.message
318
+ raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
319
+ end
238
320
 
321
+ def log_down_detected
239
322
  @last_down_at = Time.now
240
323
 
241
324
  if @down_at
242
325
  time = Time.now - @down_at
243
- Dalli.logger.debug { "#{name} is still down (for %.3f seconds now)" % time }
326
+ Dalli.logger.debug { format('%<name>s is still down (for %<time>.3f seconds now)', name: name, time: time) }
244
327
  else
245
328
  @down_at = @last_down_at
246
- Dalli.logger.warn { "#{name} is down" }
329
+ Dalli.logger.warn("#{name} is down")
247
330
  end
331
+ end
248
332
 
249
- @error = $!&.class&.name
250
- @msg ||= $!&.message
251
- raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
333
+ def log_up_detected
334
+ return unless @down_at
335
+
336
+ time = Time.now - @down_at
337
+ Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
252
338
  end
253
339
 
254
340
  def up!
255
- if @down_at
256
- time = Time.now - @down_at
257
- Dalli.logger.warn { "#{name} is back (downtime was %.3f seconds)" % time }
258
- end
341
+ log_up_detected
342
+ reset_down_info
343
+ end
259
344
 
345
+ def reset_down_info
260
346
  @fail_count = 0
261
347
  @down_at = nil
262
348
  @last_down_at = nil
@@ -268,96 +354,103 @@ module Dalli
268
354
  Thread.current[:dalli_multi]
269
355
  end
270
356
 
357
+ def cache_nils?(opts)
358
+ return false unless opts.is_a?(Hash)
359
+
360
+ opts[:cache_nils] ? true : false
361
+ end
362
+
271
363
  def get(key, options = nil)
272
- req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
364
+ req = RequestFormatter.standard_request(opkey: :get, key: key)
273
365
  write(req)
274
- generic_response(true, !!(options && options.is_a?(Hash) && options[:cache_nils]))
366
+ @response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
275
367
  end
276
368
 
277
369
  def send_multiget(keys)
278
- req = +""
370
+ req = +''
279
371
  keys.each do |key|
280
- req << [REQUEST, OPCODES[:getkq], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:getkq])
372
+ req << RequestFormatter.standard_request(opkey: :getkq, key: key)
281
373
  end
282
374
  # Could send noop here instead of in multi_response_start
283
375
  write(req)
284
376
  end
285
377
 
286
378
  def set(key, value, ttl, cas, options)
287
- (value, flags) = serialize(key, value, options)
288
- ttl = sanitize_ttl(ttl)
289
-
290
- guard_max_value(key, value)
291
-
292
- req = [REQUEST, OPCODES[multi? ? :setq : :set], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:set])
293
- write(req)
294
- cas_response unless multi?
379
+ opkey = multi? ? :setq : :set
380
+ process_value_req(opkey, key, value, ttl, cas, options)
295
381
  end
296
382
 
297
383
  def add(key, value, ttl, options)
298
- (value, flags) = serialize(key, value, options)
299
- ttl = sanitize_ttl(ttl)
300
-
301
- guard_max_value(key, value)
302
-
303
- req = [REQUEST, OPCODES[multi? ? :addq : :add], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:add])
304
- write(req)
305
- cas_response unless multi?
384
+ opkey = multi? ? :addq : :add
385
+ cas = 0
386
+ process_value_req(opkey, key, value, ttl, cas, options)
306
387
  end
307
388
 
308
389
  def replace(key, value, ttl, cas, options)
309
- (value, flags) = serialize(key, value, options)
310
- ttl = sanitize_ttl(ttl)
390
+ opkey = multi? ? :replaceq : :replace
391
+ process_value_req(opkey, key, value, ttl, cas, options)
392
+ end
311
393
 
312
- guard_max_value(key, value)
394
+ # rubocop:disable Metrics/ParameterLists
395
+ def process_value_req(opkey, key, value, ttl, cas, options)
396
+ (value, bitflags) = @value_marshaller.store(key, value, options)
397
+ ttl = TtlSanitizer.sanitize(ttl)
313
398
 
314
- req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:replace])
399
+ req = RequestFormatter.standard_request(opkey: opkey, key: key,
400
+ value: value, bitflags: bitflags,
401
+ ttl: ttl, cas: cas)
315
402
  write(req)
316
- cas_response unless multi?
403
+ @response_processor.cas_response unless multi?
317
404
  end
405
+ # rubocop:enable Metrics/ParameterLists
318
406
 
319
407
  def delete(key, cas)
320
- req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, cas, key].pack(FORMAT[:delete])
408
+ opkey = multi? ? :deleteq : :delete
409
+ req = RequestFormatter.standard_request(opkey: opkey, key: key, cas: cas)
321
410
  write(req)
322
- generic_response unless multi?
411
+ @response_processor.generic_response unless multi?
323
412
  end
324
413
 
325
- def flush(ttl)
326
- req = [REQUEST, OPCODES[:flush], 0, 4, 0, 0, 4, 0, 0, 0].pack(FORMAT[:flush])
414
+ def flush(ttl = 0)
415
+ req = RequestFormatter.standard_request(opkey: :flush, ttl: ttl)
327
416
  write(req)
328
- generic_response
417
+ @response_processor.generic_response
329
418
  end
330
419
 
331
- def decr_incr(opcode, key, count, ttl, default)
332
- expiry = default ? sanitize_ttl(ttl) : 0xFFFFFFFF
333
- default ||= 0
334
- (h, l) = split(count)
335
- (dh, dl) = split(default)
336
- req = [REQUEST, OPCODES[opcode], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[opcode])
337
- write(req)
338
- body = generic_response
339
- body ? body.unpack1("Q>") : body
420
+ # This allows us to special case a nil initial value, and
421
+ # handle it differently than a zero. This special value
422
+ # for expiry causes memcached to return a not found
423
+ # if the key doesn't already exist, rather than
424
+ # setting the initial value
425
+ NOT_FOUND_EXPIRY = 0xFFFFFFFF
426
+
427
+ def decr_incr(opkey, key, count, ttl, initial)
428
+ expiry = initial ? TtlSanitizer.sanitize(ttl) : NOT_FOUND_EXPIRY
429
+ initial ||= 0
430
+ write(RequestFormatter.decr_incr_request(opkey: opkey, key: key,
431
+ count: count, initial: initial, expiry: expiry))
432
+ @response_processor.decr_incr_response
340
433
  end
341
434
 
342
- def decr(key, count, ttl, default)
343
- decr_incr :decr, key, count, ttl, default
435
+ def decr(key, count, ttl, initial)
436
+ decr_incr :decr, key, count, ttl, initial
344
437
  end
345
438
 
346
- def incr(key, count, ttl, default)
347
- decr_incr :incr, key, count, ttl, default
439
+ def incr(key, count, ttl, initial)
440
+ decr_incr :incr, key, count, ttl, initial
348
441
  end
349
442
 
350
- def write_append_prepend(opcode, key, value)
351
- write_generic [REQUEST, OPCODES[opcode], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[opcode])
443
+ def write_append_prepend(opkey, key, value)
444
+ write_generic RequestFormatter.standard_request(opkey: opkey, key: key, value: value)
352
445
  end
353
446
 
354
447
  def write_generic(bytes)
355
448
  write(bytes)
356
- generic_response
449
+ @response_processor.generic_response
357
450
  end
358
451
 
359
452
  def write_noop
360
- req = [REQUEST, OPCODES[:noop], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
453
+ req = RequestFormatter.standard_request(opkey: :noop)
361
454
  write(req)
362
455
  end
363
456
 
@@ -365,7 +458,7 @@ module Dalli
365
458
  # We need to read all the responses at once.
366
459
  def noop
367
460
  write_noop
368
- multi_response
461
+ @response_processor.multi_with_keys_response
369
462
  end
370
463
 
371
464
  def append(key, value)
@@ -376,209 +469,36 @@ module Dalli
376
469
  write_append_prepend :prepend, key, value
377
470
  end
378
471
 
379
- def stats(info = "")
380
- req = [REQUEST, OPCODES[:stat], info.bytesize, 0, 0, 0, info.bytesize, 0, 0, info].pack(FORMAT[:stat])
472
+ def stats(info = '')
473
+ req = RequestFormatter.standard_request(opkey: :stat, key: info)
381
474
  write(req)
382
- keyvalue_response
475
+ @response_processor.multi_with_keys_response
383
476
  end
384
477
 
385
478
  def reset_stats
386
- write_generic [REQUEST, OPCODES[:stat], "reset".bytesize, 0, 0, 0, "reset".bytesize, 0, 0, "reset"].pack(FORMAT[:stat])
479
+ write_generic RequestFormatter.standard_request(opkey: :stat, key: 'reset')
387
480
  end
388
481
 
389
482
  def cas(key)
390
- req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
483
+ req = RequestFormatter.standard_request(opkey: :get, key: key)
391
484
  write(req)
392
- data_cas_response
485
+ @response_processor.data_cas_response
393
486
  end
394
487
 
395
488
  def version
396
- write_generic [REQUEST, OPCODES[:version], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
489
+ write_generic RequestFormatter.standard_request(opkey: :version)
397
490
  end
398
491
 
399
492
  def touch(key, ttl)
400
- ttl = sanitize_ttl(ttl)
401
- write_generic [REQUEST, OPCODES[:touch], key.bytesize, 4, 0, 0, key.bytesize + 4, 0, 0, ttl, key].pack(FORMAT[:touch])
493
+ ttl = TtlSanitizer.sanitize(ttl)
494
+ write_generic RequestFormatter.standard_request(opkey: :touch, key: key, ttl: ttl)
402
495
  end
403
496
 
404
497
  def gat(key, ttl, options = nil)
405
- ttl = sanitize_ttl(ttl)
406
- req = [REQUEST, OPCODES[:gat], key.bytesize, 4, 0, 0, key.bytesize + 4, 0, 0, ttl, key].pack(FORMAT[:gat])
498
+ ttl = TtlSanitizer.sanitize(ttl)
499
+ req = RequestFormatter.standard_request(opkey: :gat, key: key, ttl: ttl)
407
500
  write(req)
408
- generic_response(true, !!(options && options.is_a?(Hash) && options[:cache_nils]))
409
- end
410
-
411
- # http://www.hjp.at/zettel/m/memcached_flags.rxml
412
- # Looks like most clients use bit 0 to indicate native language serialization
413
- # and bit 1 to indicate gzip compression.
414
- FLAG_SERIALIZED = 0x1
415
- FLAG_COMPRESSED = 0x2
416
-
417
- def serialize(key, value, options = nil)
418
- marshalled = false
419
- value = if options && options[:raw]
420
- value.to_s
421
- else
422
- marshalled = true
423
- begin
424
- serializer.dump(value)
425
- rescue Timeout::Error => e
426
- raise e
427
- rescue => ex
428
- # Marshalling can throw several different types of generic Ruby exceptions.
429
- # Convert to a specific exception so we can special case it higher up the stack.
430
- exc = Dalli::MarshalError.new(ex.message)
431
- exc.set_backtrace ex.backtrace
432
- raise exc
433
- end
434
- end
435
- compressed = false
436
- set_compress_option = true if options && options[:compress]
437
- if (@options[:compress] || set_compress_option) && value.bytesize >= @options[:compression_min_size]
438
- value = compressor.compress(value)
439
- compressed = true
440
- end
441
-
442
- flags = 0
443
- flags |= FLAG_COMPRESSED if compressed
444
- flags |= FLAG_SERIALIZED if marshalled
445
- [value, flags]
446
- end
447
-
448
- def deserialize(value, flags)
449
- value = compressor.decompress(value) if (flags & FLAG_COMPRESSED) != 0
450
- value = serializer.load(value) if (flags & FLAG_SERIALIZED) != 0
451
- value
452
- rescue TypeError
453
- raise unless /needs to have method `_load'|exception class\/object expected|instance of IO needed|incompatible marshal file format/.match?($!.message)
454
- raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
455
- rescue ArgumentError
456
- raise unless /undefined class|marshal data too short/.match?($!.message)
457
- raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
458
- rescue NameError
459
- raise unless /uninitialized constant/.match?($!.message)
460
- raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
461
- rescue Zlib::Error
462
- raise UnmarshalError, "Unable to uncompress value: #{$!.message}"
463
- end
464
-
465
- def data_cas_response
466
- (extras, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
467
- data = read(count) if count > 0
468
- if status == 1
469
- nil
470
- elsif status != 0
471
- raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
472
- elsif data
473
- flags = data[0...extras].unpack1("N")
474
- value = data[extras..-1]
475
- data = deserialize(value, flags)
476
- end
477
- [data, cas]
478
- end
479
-
480
- CAS_HEADER = "@4CCnNNQ"
481
- NORMAL_HEADER = "@4CCnN"
482
- KV_HEADER = "@2n@6nN@16Q"
483
-
484
- def guard_max_value(key, value)
485
- return if value.bytesize <= @options[:value_max_bytes]
486
-
487
- message = "Value for #{key} over max size: #{@options[:value_max_bytes]} <= #{value.bytesize}"
488
- raise Dalli::ValueOverMaxSize, message
489
- end
490
-
491
- # https://github.com/memcached/memcached/blob/master/doc/protocol.txt#L79
492
- # > An expiration time, in seconds. Can be up to 30 days. After 30 days, is treated as a unix timestamp of an exact date.
493
- MAX_ACCEPTABLE_EXPIRATION_INTERVAL = 30 * 24 * 60 * 60 # 30 days
494
- def sanitize_ttl(ttl)
495
- ttl_as_i = ttl.to_i
496
- return ttl_as_i if ttl_as_i <= MAX_ACCEPTABLE_EXPIRATION_INTERVAL
497
- now = Time.now.to_i
498
- return ttl_as_i if ttl_as_i > now # already a timestamp
499
- Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached, converting to an expiration timestamp"
500
- now + ttl_as_i
501
- end
502
-
503
- # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
504
- class NilObject; end
505
- NOT_FOUND = NilObject.new
506
-
507
- def generic_response(unpack = false, cache_nils = false)
508
- (extras, _, status, count) = read_header.unpack(NORMAL_HEADER)
509
- data = read(count) if count > 0
510
- if status == 1
511
- cache_nils ? NOT_FOUND : nil
512
- elsif status == 2 || status == 5
513
- false # Not stored, normal status for add operation
514
- elsif status != 0
515
- raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
516
- elsif data
517
- flags = data.byteslice(0, extras).unpack1("N")
518
- value = data.byteslice(extras, data.bytesize - extras)
519
- unpack ? deserialize(value, flags) : value
520
- else
521
- true
522
- end
523
- end
524
-
525
- def cas_response
526
- (_, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
527
- read(count) if count > 0 # this is potential data that we don't care about
528
- if status == 1
529
- nil
530
- elsif status == 2 || status == 5
531
- false # Not stored, normal status for add operation
532
- elsif status != 0
533
- raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
534
- else
535
- cas
536
- end
537
- end
538
-
539
- def keyvalue_response
540
- hash = {}
541
- loop do
542
- (key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
543
- return hash if key_length == 0
544
- key = read(key_length)
545
- value = read(body_length - key_length) if body_length - key_length > 0
546
- hash[key] = value
547
- end
548
- end
549
-
550
- def multi_response
551
- hash = {}
552
- loop do
553
- (key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
554
- return hash if key_length == 0
555
- flags = read(4).unpack1("N")
556
- key = read(key_length)
557
- value = read(body_length - key_length - 4) if body_length - key_length - 4 > 0
558
- hash[key] = deserialize(value, flags)
559
- end
560
- end
561
-
562
- def write(bytes)
563
- @inprogress = true
564
- result = @sock.write(bytes)
565
- @inprogress = false
566
- result
567
- rescue SystemCallError, Timeout::Error => e
568
- failure!(e)
569
- end
570
-
571
- def read(count)
572
- @inprogress = true
573
- data = @sock.readfull(count)
574
- @inprogress = false
575
- data
576
- rescue SystemCallError, Timeout::Error, EOFError => e
577
- failure!(e)
578
- end
579
-
580
- def read_header
581
- read(24) || raise(Dalli::NetworkError, "No response")
501
+ @response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
582
502
  end
583
503
 
584
504
  def connect
@@ -586,13 +506,9 @@ module Dalli
586
506
 
587
507
  begin
588
508
  @pid = Process.pid
589
- @sock = if socket_type == :unix
590
- Dalli::Socket::UNIX.open(hostname, self, options)
591
- else
592
- Dalli::Socket::TCP.open(hostname, port, self, options)
593
- end
594
- sasl_authentication if need_auth?
595
- @version = version # trigger actual connect
509
+ @sock = memcached_socket
510
+ authenticate_connection if require_auth?
511
+ @version = version # Connect socket if not authed
596
512
  up!
597
513
  rescue Dalli::DalliError # SASL auth failure
598
514
  raise
@@ -602,152 +518,27 @@ module Dalli
602
518
  end
603
519
  end
604
520
 
605
- def split(n)
606
- [n >> 32, 0xFFFFFFFF & n]
607
- end
608
-
609
- REQUEST = 0x80
610
- RESPONSE = 0x81
611
-
612
- # Response codes taken from:
613
- # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
614
- RESPONSE_CODES = {
615
- 0 => "No error",
616
- 1 => "Key not found",
617
- 2 => "Key exists",
618
- 3 => "Value too large",
619
- 4 => "Invalid arguments",
620
- 5 => "Item not stored",
621
- 6 => "Incr/decr on a non-numeric value",
622
- 7 => "The vbucket belongs to another server",
623
- 8 => "Authentication error",
624
- 9 => "Authentication continue",
625
- 0x20 => "Authentication required",
626
- 0x81 => "Unknown command",
627
- 0x82 => "Out of memory",
628
- 0x83 => "Not supported",
629
- 0x84 => "Internal error",
630
- 0x85 => "Busy",
631
- 0x86 => "Temporary failure"
632
- }
633
-
634
- OPCODES = {
635
- get: 0x00,
636
- set: 0x01,
637
- add: 0x02,
638
- replace: 0x03,
639
- delete: 0x04,
640
- incr: 0x05,
641
- decr: 0x06,
642
- flush: 0x08,
643
- noop: 0x0A,
644
- version: 0x0B,
645
- getkq: 0x0D,
646
- append: 0x0E,
647
- prepend: 0x0F,
648
- stat: 0x10,
649
- setq: 0x11,
650
- addq: 0x12,
651
- replaceq: 0x13,
652
- deleteq: 0x14,
653
- incrq: 0x15,
654
- decrq: 0x16,
655
- auth_negotiation: 0x20,
656
- auth_request: 0x21,
657
- auth_continue: 0x22,
658
- touch: 0x1C,
659
- gat: 0x1D
660
- }
661
-
662
- HEADER = "CCnCCnNNQ"
663
- OP_FORMAT = {
664
- get: "a*",
665
- set: "NNa*a*",
666
- add: "NNa*a*",
667
- replace: "NNa*a*",
668
- delete: "a*",
669
- incr: "NNNNNa*",
670
- decr: "NNNNNa*",
671
- flush: "N",
672
- noop: "",
673
- getkq: "a*",
674
- version: "",
675
- stat: "a*",
676
- append: "a*a*",
677
- prepend: "a*a*",
678
- auth_request: "a*a*",
679
- auth_continue: "a*a*",
680
- touch: "Na*",
681
- gat: "Na*"
682
- }
683
- FORMAT = OP_FORMAT.each_with_object({}) { |(k, v), memo| memo[k] = HEADER + v; }
684
-
685
- #######
686
- # SASL authentication support for NorthScale
687
- #######
688
-
689
- def need_auth?
690
- @options[:username] || ENV["MEMCACHE_USERNAME"]
521
+ def memcached_socket
522
+ if socket_type == :unix
523
+ Dalli::Socket::UNIX.open(hostname, self, options)
524
+ else
525
+ Dalli::Socket::TCP.open(hostname, port, self, options)
526
+ end
527
+ end
528
+
529
+ def require_auth?
530
+ !username.nil?
691
531
  end
692
532
 
693
533
  def username
694
- @options[:username] || ENV["MEMCACHE_USERNAME"]
534
+ @options[:username] || ENV['MEMCACHE_USERNAME']
695
535
  end
696
536
 
697
537
  def password
698
- @options[:password] || ENV["MEMCACHE_PASSWORD"]
538
+ @options[:password] || ENV['MEMCACHE_PASSWORD']
699
539
  end
700
540
 
701
- def sasl_authentication
702
- Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
703
-
704
- # negotiate
705
- req = [REQUEST, OPCODES[:auth_negotiation], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
706
- write(req)
707
-
708
- (extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
709
- raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
710
- content = read(count).tr("\u0000", " ")
711
- return Dalli.logger.debug("Authentication not required/supported by server") if status == 0x81
712
- mechanisms = content.split(" ")
713
- raise NotImplementedError, "Dalli only supports the PLAIN authentication mechanism" unless mechanisms.include?("PLAIN")
714
-
715
- # request
716
- mechanism = "PLAIN"
717
- msg = "\x0#{username}\x0#{password}"
718
- req = [REQUEST, OPCODES[:auth_request], mechanism.bytesize, 0, 0, 0, mechanism.bytesize + msg.bytesize, 0, 0, mechanism, msg].pack(FORMAT[:auth_request])
719
- write(req)
720
-
721
- (extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
722
- raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
723
- content = read(count)
724
- return Dalli.logger.info("Dalli/SASL: #{content}") if status == 0
725
-
726
- raise Dalli::DalliError, "Error authenticating: #{status}" unless status == 0x21
727
- raise NotImplementedError, "No two-step authentication mechanisms supported"
728
- # (step, msg) = sasl.receive('challenge', content)
729
- # raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
730
- end
731
-
732
- def parse_hostname(str)
733
- res = str.match(/\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/)
734
- raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == "[]"
735
- hostnam = res[2] || res[1]
736
- if hostnam.start_with?("/")
737
- socket_type = :unix
738
- # in case of unix socket, allow only setting of weight, not port
739
- raise Dalli::DalliError, "Could not parse hostname #{str}" if res[4]
740
- weigh = res[3]
741
- else
742
- socket_type = :tcp
743
- por = res[3] || DEFAULT_PORT
744
- por = Integer(por)
745
- weigh = res[4]
746
- end
747
- weigh ||= DEFAULT_WEIGHT
748
- weigh = Integer(weigh)
749
- [hostnam, por, weigh, socket_type]
750
- end
541
+ include SaslAuthentication
751
542
  end
752
543
  end
753
544
  end