dalli 5.2.2 → 5.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -43,6 +43,7 @@ module Dalli
43
43
  @options = client_options.merge(user_creds)
44
44
  @raw_mode = client_options[:raw]
45
45
  @defer_drain = client_options[:defer_drain] ? true : false
46
+ @verify_reply_keys = client_options[:verify_reply_keys] ? true : false
46
47
  @value_marshaller = @raw_mode ? StringMarshaller.new(@options) : ValueMarshaller.new(@options)
47
48
  @connection_manager = ConnectionManager.new(hostname, port, socket_type, @options)
48
49
  end
@@ -68,7 +69,7 @@ module Dalli
68
69
  begin
69
70
  request_completed = false
70
71
  @connection_manager.start_request!
71
- drain_deferred_quiet_responses(opkey)
72
+ drain_pending_quiet_responses(opkey)
72
73
  response = send(opkey, *args)
73
74
  # Quiet requests can leave replies on the socket; remember that so the
74
75
  # end of the quiet block knows this server needs draining
@@ -79,8 +80,14 @@ module Dalli
79
80
 
80
81
  request_completed = true
81
82
  response
82
- rescue Dalli::MarshalError => e
83
- log_marshal_err(args.first, e)
83
+ rescue Dalli::MarshalError, Dalli::ValueOverMaxSize => e
84
+ log_marshal_err(args.first, e) if e.is_a?(Dalli::MarshalError)
85
+ # Raised only while the connection is clean: a value is marshalled
86
+ # before any of its request is written, and write_multi_req raises
87
+ # only after reading every reply. Closing the connection would
88
+ # discard earlier quiet writes still waiting in the write buffer.
89
+ @connection_manager.abort_request!
90
+ request_completed = true
84
91
  raise
85
92
  rescue Dalli::DalliError
86
93
  raise
@@ -140,7 +147,7 @@ module Dalli
140
147
  verify_pipelined_state(:getkq)
141
148
  write_noop
142
149
  # Use ensure_ready instead of reset to preserve any data already buffered
143
- # during interleaved pipelined get draining
150
+ # while this get_multi's interleaved requests were sent
144
151
  response_buffer.ensure_ready
145
152
  end
146
153
 
@@ -153,6 +160,8 @@ module Dalli
153
160
  # avoiding intermediate Hash allocation. Returns nil.
154
161
  # Without a block, returns a Hash of { key => [value, cas] }.
155
162
  def pipeline_next_responses(&block)
163
+ return abandon_pipeline if @pipeline_out_of_step
164
+
156
165
  reconnect_on_pipeline_complete!
157
166
  values = nil
158
167
 
@@ -174,6 +183,13 @@ module Dalli
174
183
  # to the noop at the end of the pipeline
175
184
  finish_pipeline && break if status && key.nil?
176
185
 
186
+ # A key that wasn't requested means the replies are out of step
187
+ # (see expect_reply_keys): keep what was read so far, drop the rest
188
+ if unexpected_reply_key?(key)
189
+ abandon_pipeline
190
+ break
191
+ end
192
+
177
193
  # If the status is ok and the key is not nil, then this is a
178
194
  # getkq response with a value that we want to set in the response hash
179
195
  unless key.nil?
@@ -185,6 +201,10 @@ module Dalli
185
201
  end
186
202
  end
187
203
 
204
+ # A request to this server from inside the block closes the
205
+ # connection, and with it the rest of this get_multi's replies
206
+ raise_pipeline_closed unless response_buffer.in_progress?
207
+
188
208
  # Get the next response from the buffer
189
209
  status, cas, key, value = response_buffer.process_single_getk_response
190
210
  end
@@ -194,21 +214,24 @@ module Dalli
194
214
  # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
195
215
 
196
216
  # Abort current pipelined get. Generally used to signal an external
197
- # timeout during pipelined get. The underlying socket is
217
+ # timeout during pipelined get, or that the get_multi stopped before
218
+ # reading all its replies (timed_out: false). The underlying socket is
198
219
  # disconnected, and the exception is swallowed.
199
220
  #
200
221
  # Returns nothing.
201
- def pipeline_abort
222
+ def pipeline_abort(timed_out: true)
202
223
  response_buffer.clear
224
+ @pipeline_out_of_step = false
225
+ @expected_reply_keys = nil
203
226
  @connection_manager.abort_request!
204
227
  return true unless connected?
205
228
 
206
229
  # Closes the connection, which ensures that our connection is in a
207
230
  # clean state for future requests. The get_multi ran out of time
208
- # overall rather than hitting a socket error, so it doesn't count
209
- # toward socket_max_failures: two slow get_multis in a row would
210
- # otherwise mark a healthy server down.
211
- Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
231
+ # overall, or stopped early, rather than hitting a socket error, so it
232
+ # doesn't count toward socket_max_failures: two slow get_multis in a
233
+ # row would otherwise mark a healthy server down.
234
+ log_pipeline_abort(timed_out)
212
235
  @connection_manager.close
213
236
  true
214
237
  end
@@ -240,10 +263,15 @@ module Dalli
240
263
  ALLOWED_QUIET_OPS = %i[add replace set delete incr decr append prepend flush noop].freeze
241
264
  private_constant :ALLOWED_QUIET_OPS
242
265
 
243
- # With defer_drain, quiet blocks leave their replies unread. Read them
244
- # before any non-quiet request, which reads a reply of its own.
245
- def drain_deferred_quiet_responses(opkey)
246
- return unless @defer_drain && !quiet? && opkey != :noop && @connection_manager.quiet_responses_pending?
266
+ # Reads the replies still pending from quiet requests before any
267
+ # non-quiet request, which reads a reply of its own. With defer_drain,
268
+ # quiet blocks leave their replies unread. Without it, a quiet block
269
+ # reads them at its end, but the connection is shared: another thread or
270
+ # fiber (quiet? is fiber-local) can make a request while the block is
271
+ # still open, and would otherwise read the block's replies as its own.
272
+ # The noop also sends any quiet requests still buffered.
273
+ def drain_pending_quiet_responses(opkey)
274
+ return unless @connection_manager.quiet_responses_pending? && opkey != :noop && !quiet?
247
275
 
248
276
  noop
249
277
  end
@@ -346,6 +374,7 @@ module Dalli
346
374
  # the expected state before pipeline_response_setup is called.
347
375
  response_buffer.clear
348
376
  response_buffer.raw = raw_request?(options)
377
+ expect_reply_keys(keys)
349
378
 
350
379
  # The terminating noop is sent by pipeline_response_setup
351
380
  write(quiet_get_requests(keys, options, return_cas: return_cas))
@@ -355,9 +384,11 @@ module Dalli
355
384
  # This prevents socket buffer deadlock when sending many keys.
356
385
  # Populates the provided results hash with any responses drained during send.
357
386
  def pipelined_get_interleaved(keys, chunk_size, results, options = nil)
358
- # Initialize the response buffer for draining during send phase
359
- response_buffer.ensure_ready
387
+ # Start with an empty buffer for draining during the send phase. Bytes
388
+ # left by an earlier get_multi that stopped early belong to its replies.
389
+ response_buffer.reset
360
390
  response_buffer.raw = raw_request?(options)
391
+ expect_reply_keys(keys)
361
392
 
362
393
  keys.each_slice(chunk_size) do |chunk|
363
394
  # Build and write this chunk of requests
@@ -390,6 +421,12 @@ module Dalli
390
421
  status, cas, key, value = response_buffer.process_single_getk_response
391
422
  break if status.nil? # No complete response available
392
423
 
424
+ # Handled by the next pipeline_next_responses, once sending is done
425
+ if unexpected_reply_key?(key)
426
+ @pipeline_out_of_step = true
427
+ break
428
+ end
429
+
393
430
  results[key] = [value, cas] unless key.nil?
394
431
  end
395
432
  rescue SystemCallError, Dalli::NetworkError
@@ -397,8 +434,48 @@ module Dalli
397
434
  nil
398
435
  end
399
436
 
437
+ # Registered with the connection manager, which clears it whenever the
438
+ # connection closes, so a new connection never parses an old one's bytes
400
439
  def response_buffer
401
- @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
440
+ @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor).tap do |buffer|
441
+ @connection_manager.response_buffer = buffer
442
+ end
443
+ end
444
+
445
+ # A multi-get only accepts replies for the keys it asked for. memcached
446
+ # names an item created by auto-create (ma or mg with N) for a
447
+ # base64-encoded key by its raw bytes, without the b flag; such a key
448
+ # can contain spaces or CR/LF, which split the reply header and make the
449
+ # replies after it parse as different ones. A key that wasn't requested
450
+ # is how that shows up.
451
+ # Only with the verify_reply_keys option, since building the set costs a
452
+ # multi-get about 15%. Dalli itself no longer creates such items (see
453
+ # Meta#decr_incr); the check guards against ones that older versions or
454
+ # other clients created, on memcached versions with the behavior.
455
+ def expect_reply_keys(keys)
456
+ @pipeline_out_of_step = false
457
+ @expected_reply_keys = (keys.to_set { |k| reply_key_form(k) } if @verify_reply_keys)
458
+ end
459
+
460
+ def unexpected_reply_key?(key)
461
+ !key.nil? && @expected_reply_keys && !@expected_reply_keys.include?(reply_key_form(key))
462
+ end
463
+
464
+ # Keys compared as bytes; ASCII keys compare equal in any encoding as is
465
+ def reply_key_form(key)
466
+ key.ascii_only? ? key : key.b
467
+ end
468
+
469
+ # Keeps what was read, drops the rest of this server's replies, and
470
+ # closes the connection so nothing unread is left on it. Not a socket
471
+ # failure, so it doesn't count toward socket_max_failures; the keys not
472
+ # read are simply missing from the result.
473
+ def abandon_pipeline
474
+ Dalli.logger.warn { "#{name} replied to a get_multi with a key that wasn't requested; closing the connection" }
475
+ @pipeline_out_of_step = false
476
+ response_buffer.clear
477
+ @connection_manager.close
478
+ {}
402
479
  end
403
480
 
404
481
  # Called after the noop response is received at the end of a set
@@ -414,7 +491,23 @@ module Dalli
414
491
  @connection_manager.reconnect! 'pipelined get has completed' if pipeline_complete?
415
492
  end
416
493
 
494
+ def raise_pipeline_closed
495
+ raise Dalli::DalliError, "The connection to #{name} closed while a get_multi was reading its replies " \
496
+ '(a request made from inside the get_multi block?)'
497
+ end
498
+
499
+ def log_pipeline_abort(timed_out)
500
+ if timed_out
501
+ Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
502
+ else
503
+ Dalli.logger.debug { "#{name} get_multi stopped before reading all replies; closing the connection" }
504
+ end
505
+ end
506
+
507
+ # The single-server set_multi path passes its whole pairs hash as the
508
+ # first argument; log only its keys, never the values
417
509
  def log_marshal_err(key, err)
510
+ key = key.keys.join("', '") if key.is_a?(Hash)
418
511
  Dalli.logger.error "Marshalling error for key '#{key}': #{err.message}"
419
512
  Dalli.logger.error 'You are trying to cache a Ruby object which cannot be serialized to memcached.'
420
513
  end
@@ -26,6 +26,9 @@ module Dalli
26
26
 
27
27
  attr_accessor :hostname, :port, :socket_type, :options
28
28
  attr_reader :sock
29
+ # The buffer replies are read into (see Base#response_buffer). It's
30
+ # cleared with the write buffer whenever the connection closes.
31
+ attr_writer :response_buffer
29
32
 
30
33
  def initialize(hostname, port, socket_type, client_options)
31
34
  @hostname = hostname
@@ -38,6 +41,7 @@ module Dalli
38
41
  @pid = nil
39
42
  @write_buffer = []
40
43
  @write_buffer_bytes = 0
44
+ @response_buffer = nil
41
45
 
42
46
  @fail_count = 0
43
47
  reset_down_info
@@ -60,8 +64,8 @@ module Dalli
60
64
  @pid = Process.pid
61
65
  @request_in_progress = false
62
66
  @quiet_responses_pending = false
63
- rescue SystemCallError, *TIMEOUT_ERRORS, EOFError, SocketError => e
64
- # SocketError = DNS resolution failure
67
+ rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError, SocketError => e
68
+ # SocketError = DNS resolution failure; SSL_ERRORS = failed TLS handshake
65
69
  error_on_request!(e)
66
70
  end
67
71
 
@@ -133,6 +137,7 @@ module Dalli
133
137
  @pid = nil
134
138
  @quiet_responses_pending = false
135
139
  discard_write_buffer
140
+ @response_buffer&.clear
136
141
  abort_request!
137
142
  end
138
143
  end
@@ -188,10 +193,22 @@ module Dalli
188
193
  @request_in_progress = false
189
194
  end
190
195
 
196
+ # The longest reply line Dalli reads, in bytes, CRLF included. A meta
197
+ # reply header is at most about 500 bytes: "VA <size>" plus its flags,
198
+ # the largest being a 250-byte key base64-encoded to 336 bytes, a
199
+ # 20-digit CAS, a 32-byte opaque and a few numeric fields. The longest
200
+ # stats line (stats settings) carries a file path such as ext_path, up
201
+ # to PATH_MAX (4096). 8 KiB leaves room for both, while a server that
202
+ # never sends CRLF can't grow the read buffer without bound.
203
+ MAX_LINE_BYTES = 8 * 1024
204
+
191
205
  def read_line
192
206
  flush_write_buffer
193
- data = @sock.gets("\r\n")
207
+ data = @sock.gets("\r\n", MAX_LINE_BYTES)
194
208
  error_on_request!('EOF in read_line') if data.nil?
209
+ # Without its CRLF the line either ran past MAX_LINE_BYTES or was cut
210
+ # short by EOF; either way the connection is out of step
211
+ error_on_request!("Reply line without CRLF in the first #{MAX_LINE_BYTES} bytes") unless data.end_with?("\r\n")
195
212
  data
196
213
  rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
197
214
  error_on_request!(e)
@@ -234,12 +234,45 @@ module Dalli
234
234
  # rubocop:disable-next Metrics/ParameterLists
235
235
  def decr_incr(incr, key, delta, ttl, initial, options = nil)
236
236
  ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
237
+ # memcached stores an item created by ma's auto-create (N/J) for a
238
+ # base64-encoded key without its binary-key flag, and later replies
239
+ # name it with the raw key bytes. Create such counters with an add.
240
+ return decr_incr_with_add(incr, key, delta, ttl, initial, options) if initial && KeyRegularizer.required?(key)
241
+
237
242
  write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: initial, incr: incr, ttl: ttl,
238
243
  quiet: quiet?, **routing_token_kwargs(options)))
239
244
  finish_write(quiet?)
240
245
  response_processor.decr_incr unless quiet?
241
246
  end
242
247
 
248
+ # Like decr_incr with an initial value, without memcached's auto-create:
249
+ # on a miss the counter is added with the initial value (which is
250
+ # returned, as auto-create would), and the add losing a race to another
251
+ # client means the counter now exists, so the incr is tried again. In a
252
+ # quiet block both requests are sent and whichever applies takes effect.
253
+ # rubocop:disable-next Metrics/ParameterLists
254
+ def decr_incr_with_add(incr, key, delta, ttl, initial, options)
255
+ add_options = (options || {}).merge(raw: true) # counters are stored raw
256
+ return quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options) if quiet?
257
+
258
+ 2.times do
259
+ value = decr_incr(incr, key, delta, nil, nil, options)
260
+ return value unless value.nil?
261
+
262
+ write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: false)
263
+ return initial if response_processor.meta_set_with_cas
264
+ end
265
+ decr_incr(incr, key, delta, nil, nil, options)
266
+ end
267
+
268
+ # rubocop:disable-next Metrics/ParameterLists
269
+ def quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options)
270
+ write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: nil, incr: incr, ttl: nil,
271
+ quiet: true, **routing_token_kwargs(add_options)))
272
+ write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: true)
273
+ finish_write(true)
274
+ end
275
+
243
276
  # Other Commands
244
277
  def flush(delay = 0)
245
278
  write(RequestFormatter.flush(delay: delay))
@@ -250,10 +283,12 @@ module Dalli
250
283
  # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
251
284
  # We need to read all the responses at once.
252
285
  # A write that expects a reply is flushed so the reply can be read. A write
253
- # in a quiet block stays buffered until the block's noop, unless
254
- # defer_drain is on: then no noop follows at the end of the block, so it is
255
- # flushed right away. (pipelined_set passes quiet: true outside a quiet
256
- # block; its caller flushes with a terminating noop.)
286
+ # in a quiet block stays buffered until a noop: the one at the end of the
287
+ # block, or the one another thread or fiber sends to drain the block's
288
+ # replies before its own request. With defer_drain no noop follows at the
289
+ # end of the block, so it is flushed right away. (pipelined_set passes
290
+ # quiet: true outside a quiet block; its caller flushes with a
291
+ # terminating noop.)
257
292
  def finish_write(quiet)
258
293
  @connection_manager.flush if !quiet || (@defer_drain && quiet?)
259
294
  end
@@ -293,7 +328,7 @@ module Dalli
293
328
  buffer = RequestFormatter.multi_meta_get(keys, skip_flags: is_raw, **routing_token_kwargs(options))
294
329
  flushed_write(buffer)
295
330
  buffer.clear
296
- read_multi_get_responses(is_raw)
331
+ read_multi_get_responses(is_raw, keys)
297
332
  end
298
333
 
299
334
  # Stale-aware bulk get. Returns { key => { value:, cas:, stale:, miss: } }
@@ -310,12 +345,13 @@ module Dalli
310
345
  **routing_token_kwargs(options))
311
346
  flushed_write(buffer)
312
347
  buffer.clear
313
- read_multi_metadata_responses(is_raw)
348
+ read_multi_metadata_responses(is_raw, keys)
314
349
  end
315
350
 
316
351
  # Unlike read_multi_get_responses this locates tokens by flag rather than
317
352
  # by position, because the c flag shifts the key's index.
318
- def read_multi_metadata_responses(is_raw)
353
+ def read_multi_metadata_responses(is_raw, keys)
354
+ expect_reply_keys(keys)
319
355
  hash = {}
320
356
  while (line = @connection_manager.read_line)
321
357
  break if line.start_with?('MN')
@@ -331,23 +367,34 @@ module Dalli
331
367
  key = response_processor.key_from_tokens(tokens)
332
368
  next if key.nil?
333
369
 
370
+ raise_unexpected_reply_key(key) if unexpected_reply_key?(key)
371
+
334
372
  hash[key] = { value: @value_marshaller.retrieve(value, bitflags), cas: cas, stale: stale, miss: false }
335
373
  end
336
374
  hash
337
375
  end
338
376
 
339
- def read_multi_get_responses(is_raw)
377
+ def read_multi_get_responses(is_raw, keys)
378
+ expect_reply_keys(keys)
340
379
  hash = {}
341
380
  while (line = @connection_manager.read_line)
342
381
  break if line.start_with?('MN')
343
382
  next unless line.start_with?('VA ')
344
383
 
345
384
  key, value = parse_multi_get_value(line, is_raw)
385
+ raise_unexpected_reply_key(key) if unexpected_reply_key?(key)
346
386
  hash[key] = value if key
347
387
  end
348
388
  hash
349
389
  end
350
390
 
391
+ # The replies after a key that wasn't requested can't be trusted (see
392
+ # Base#expect_reply_keys). Raising ends the request, which closes the
393
+ # connection; the caller treats this server's keys as missing.
394
+ def raise_unexpected_reply_key(key)
395
+ raise Dalli::UnexpectedReplyKeyError, "#{name} replied with a key that wasn't requested: #{key.inspect}"
396
+ end
397
+
351
398
  # Reads the "VA <size> [f<flags>] k<key> ..." line in place rather than
352
399
  # splitting it into tokens
353
400
  def parse_multi_get_value(line, is_raw)
@@ -364,16 +411,27 @@ module Dalli
364
411
 
365
412
  # Single-server fast path for set_multi. Inlines request formatting to
366
413
  # minimize per-key overhead. Avoids PipelinedSetter server grouping.
414
+ #
415
+ # A value that can't be marshalled or is over value_max_bytes is skipped;
416
+ # the other keys are still stored, then the first such error is raised
417
+ # (as PipelinedSetter does on the multi-server path).
367
418
  def write_multi_req(pairs, ttl, req_options)
368
419
  ttl = TtlSanitizer.sanitize(ttl) if ttl
369
- entries = pairs.map do |key, raw_value|
420
+ value_error = nil
421
+ entries = pairs.filter_map do |key, raw_value|
370
422
  [key, @value_marshaller.store(key, raw_value, req_options)]
423
+ rescue Dalli::MarshalError, Dalli::ValueOverMaxSize => e
424
+ value_error ||= e
425
+ nil
371
426
  end
372
427
 
373
- buffer = RequestFormatter.multi_meta_set(entries, ttl: ttl, **routing_token_kwargs(req_options))
374
- flushed_write(buffer)
375
- buffer.clear
376
- response_processor.consume_all_responses_until_mn
428
+ unless entries.empty?
429
+ buffer = RequestFormatter.multi_meta_set(entries, ttl: ttl, **routing_token_kwargs(req_options))
430
+ flushed_write(buffer)
431
+ buffer.clear
432
+ response_processor.consume_all_responses_until_mn
433
+ end
434
+ raise value_error if value_error
377
435
  end
378
436
 
379
437
  # Single-server fast path for delete_multi. Writes all quiet delete requests
@@ -33,7 +33,7 @@ module Dalli
33
33
  username: uri.user,
34
34
  password: uri.password
35
35
  }
36
- [uri.host, normalize_port(uri.port), :tcp, DEFAULT_WEIGHT, auth_details]
36
+ [uri.hostname, normalize_port(uri.port), :tcp, DEFAULT_WEIGHT, auth_details]
37
37
  end
38
38
 
39
39
  def self.parse_non_uri(str)
@@ -61,11 +61,11 @@ module Dalli
61
61
  # in case of unix socket, allow only setting of weight, not port
62
62
  raise Dalli::DalliError, "Could not parse hostname #{res[0]}" if res[4]
63
63
 
64
- [nil, normalize_weight(res[3])]
64
+ [nil, normalize_weight(res[3], res[0])]
65
65
  end
66
66
 
67
67
  def self.attributes_for_tcp_socket(res)
68
- [normalize_port(res[3]), normalize_weight(res[4])]
68
+ [normalize_port(res[3]), normalize_weight(res[4], res[0])]
69
69
  end
70
70
 
71
71
  def self.normalize_host_from_match(str, res)
@@ -78,8 +78,13 @@ module Dalli
78
78
  Integer(port || DEFAULT_PORT)
79
79
  end
80
80
 
81
- def self.normalize_weight(weight)
82
- Integer(weight || DEFAULT_WEIGHT)
81
+ # A weight below 1 gives the server no points on the ring, and if every
82
+ # server has one the ring divides by a zero total weight.
83
+ def self.normalize_weight(weight, str)
84
+ value = Integer(weight || DEFAULT_WEIGHT)
85
+ raise ArgumentError, "Server weight must be at least 1, got #{value} in #{str}" if value < 1
86
+
87
+ value
83
88
  end
84
89
  end
85
90
  end
@@ -10,7 +10,8 @@ module Dalli
10
10
  ##
11
11
  class StringMarshaller
12
12
  DEFAULTS = {
13
- # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
13
+ # max memcached item size in bytes: value + key + ITEM_OVERHEAD_BYTES. Set it to
14
+ # memcached's -I value (default 1 MB).
14
15
  value_max_bytes: 1024 * 1024
15
16
  }.freeze
16
17
 
@@ -55,10 +56,7 @@ module Dalli
55
56
  private
56
57
 
57
58
  def error_if_over_max_value_bytes(key, value)
58
- return if value.bytesize <= value_max_bytes
59
-
60
- message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
61
- raise Dalli::ValueOverMaxSize, message
59
+ ValueMarshaller.error_if_over_max_value_bytes(key, value, value_max_bytes)
62
60
  end
63
61
  end
64
62
  end
@@ -84,11 +84,13 @@ module Dalli
84
84
  end
85
85
 
86
86
  # Checks whether we should apply compression when serializing a value
87
- # based on the specified options. Returns false unless the value
88
- # is greater than the minimum compression size. Otherwise returns
89
- # based on a method-level option if specified, falling back to the
90
- # server default.
87
+ # based on the specified options. Returns false for a raw request
88
+ # (raw reads ignore bitflags, so the value must be stored as given) or
89
+ # unless the value is greater than the minimum compression size.
90
+ # Otherwise returns based on a method-level option if specified,
91
+ # falling back to the server default.
91
92
  def compress_value?(value, req_options)
93
+ return false if req_options&.dig(:raw)
92
94
  return false unless value.bytesize >= compression_min_size
93
95
  return compress_by_default? unless req_options && !req_options[:compress].nil?
94
96
 
@@ -13,12 +13,30 @@ module Dalli
13
13
  extend Forwardable
14
14
 
15
15
  DEFAULTS = {
16
- # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
16
+ # max memcached item size in bytes: value + key + ITEM_OVERHEAD_BYTES. Set it to
17
+ # memcached's -I value (default 1 MB).
17
18
  value_max_bytes: 1024 * 1024
18
19
  }.freeze
19
20
 
20
21
  OPTIONS = DEFAULTS.keys.freeze
21
22
 
23
+ # memcached's -I limit applies to the whole item, not just the value. On
24
+ # 64-bit memcached 1.6 an item takes the value, the key, and 63 bytes of
25
+ # overhead: a 48-byte header, 8 bytes of CAS, 4 bytes of client flags,
26
+ # the key's NUL terminator and the value's trailing CRLF. (Measured
27
+ # against memcached 1.6.45 with -I 512k, 1m, 2m and 5m; an item stored
28
+ # with zero flags or with CAS disabled needs a few bytes less.)
29
+ ITEM_OVERHEAD_BYTES = 63
30
+
31
+ def self.error_if_over_max_value_bytes(key, value, value_max_bytes)
32
+ item_bytes = value.bytesize + key.bytesize + ITEM_OVERHEAD_BYTES
33
+ return if item_bytes <= value_max_bytes
34
+
35
+ message = "Value for #{key} over max size: #{value_max_bytes} <= #{item_bytes} " \
36
+ "(#{value.bytesize} value bytes + #{key.bytesize} key bytes + #{ITEM_OVERHEAD_BYTES} item overhead)"
37
+ raise Dalli::ValueOverMaxSize, message
38
+ end
39
+
22
40
  def_delegators :@value_serializer, :serializer
23
41
  def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
24
42
 
@@ -28,6 +46,7 @@ module Dalli
28
46
 
29
47
  @marshal_options =
30
48
  DEFAULTS.merge(client_options.slice(*OPTIONS))
49
+ @marshal_options[:value_max_bytes] = @marshal_options[:value_max_bytes].to_i
31
50
  end
32
51
 
33
52
  def store(key, value, options = nil)
@@ -49,10 +68,7 @@ module Dalli
49
68
  end
50
69
 
51
70
  def error_if_over_max_value_bytes(key, value)
52
- return if value.bytesize <= value_max_bytes
53
-
54
- message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
55
- raise Dalli::ValueOverMaxSize, message
71
+ self.class.error_if_over_max_value_bytes(key, value, value_max_bytes)
56
72
  end
57
73
  end
58
74
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'openssl'
3
4
  require 'timeout'
4
5
 
5
6
  module Dalli
@@ -16,14 +17,11 @@ module Dalli
16
17
  [Timeout::Error]
17
18
  end
18
19
 
19
- # SSL errors that occur during read/write operations (not during initial
20
- # handshake) should trigger reconnection. These indicate transient network
21
- # issues, not configuration problems.
22
- SSL_ERRORS =
23
- if defined?(OpenSSL::SSL::SSLError)
24
- [OpenSSL::SSL::SSLError]
25
- else
26
- []
27
- end
20
+ # SSL errors during the handshake or during read/write operations count as
21
+ # a failed request: they trigger reconnection and, after
22
+ # socket_max_failures, mark the server down.
23
+ # openssl is required above: this file loads before dalli/socket, so a
24
+ # defined? check here would find OpenSSL missing and leave this empty.
25
+ SSL_ERRORS = [OpenSSL::SSL::SSLError].freeze
28
26
  end
29
27
  end