dalli 5.3.2 → 5.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 01e6a1f1483a28b5913847f35280e7c4cf2f490f01ddda3607ba1cf7687e3081
4
- data.tar.gz: 0a4c8e31094f2d0fabc9a17062765d7e5293ec29ed338fabca6c93076d4ea224
3
+ metadata.gz: afa96265afb6af5b3b8857b804cc929ca81847196c2c0f2de1c1066806fb4ab8
4
+ data.tar.gz: 45c193be95ffa8ce01562ffe0c7ada3c800f66649628275ff7c2fe64b85d9c7f
5
5
  SHA512:
6
- metadata.gz: 1fa29fe92cc42e648f5752c79a185466e24614283d0ebed7f8a12a5b3ab2f98806065c1989c403144b10809ad33baaa4033c66f140e80500c4ecb9c3a6b6a41b
7
- data.tar.gz: be9a46942962a98dc2d14d9058f48c0b78c22c245b447e7578b030aa941e286a67941516e0d8a9d377167fd9fcbb3960865c3c118e2f3e8ba88ba1aece683914
6
+ metadata.gz: 861f348e78ce13cf30b3695669c6ca32f9bf22c5800bca2d296fd9766c977bdf484856876efafa6be77ae7c7c9a85c911f461edbe44538410913d221e87481c4
7
+ data.tar.gz: bd13c339ec62d3f14040d93f61f55ad5395edbfaba9da04998fc6731b11519e7446b5d49b7b846ac4c5dcbd65a1457ca07221042dcc83e4e58408a761cecba53
data/CHANGELOG.md CHANGED
@@ -4,6 +4,13 @@ Dalli Changelog
4
4
  Unreleased
5
5
  ==========
6
6
 
7
+ 5.3.3
8
+ ==========
9
+
10
+ Security:
11
+
12
+ - Bound memcached reply headers, incoming value sizes, and response counts, and enforce TLS timeouts for blocking operations (GHSA-5wpc-45jc-6ccc)
13
+
7
14
  5.3.2
8
15
  ==========
9
16
 
data/README.md CHANGED
@@ -70,6 +70,8 @@ Dalli::Client.new('localhost:11211', value_max_bytes: 2 * 1024 * 1024)
70
70
 
71
71
  memcached's limit covers the whole item, so Dalli checks the stored (serialized and compressed) value plus the key plus 63 bytes of per-item overhead against `value_max_bytes`, and raises `Dalli::ValueOverMaxSize` before sending anything larger. With the defaults, the largest value you can store under a 10-byte key is 1,048,576 - 10 - 63 = 1,048,503 bytes.
72
72
 
73
+ Dalli also limits the size of values it reads to `value_max_bytes`. If another client writes larger items that this client needs to read, set `max_response_bytes` explicitly. Keep it no higher than the largest item size you intend to accept from the server.
74
+
73
75
  ### Deferred Draining
74
76
 
75
77
  By default, a `quiet` (or `multi`) block ends by waiting for the replies memcached still sends for its quiet requests, one round trip for each server the block wrote to. With `defer_drain: true`, the block returns without waiting:
data/lib/dalli/client.rb CHANGED
@@ -39,6 +39,8 @@ module Dalli
39
39
  # memcached, Dalli counts the key and 63 bytes of item overhead against this limit along
40
40
  # with the (serialized, compressed) value, so the largest value that can be stored is
41
41
  # value_max_bytes - key bytesize - 63. Larger values raise Dalli::ValueOverMaxSize.
42
+ # - :max_response_bytes - largest value accepted from a server. Defaults to :value_max_bytes (1MB by default).
43
+ # Set this explicitly when reading larger items written by another client.
42
44
  # - :serializer - defaults to Marshal
43
45
  # - :compressor - defaults to Dalli::Compressor, a Zlib-based implementation
44
46
  # - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for
@@ -247,6 +247,16 @@ module Dalli
247
247
  def servers_with_response(servers, timeout)
248
248
  return [] if servers.empty?
249
249
 
250
+ # A bounded read can stop while OpenSSL still holds decrypted bytes.
251
+ # IO.select only sees the underlying socket, so process those bytes
252
+ # before waiting for another network event.
253
+ buffered = nil
254
+ servers.each do |server|
255
+ sock = server.sock
256
+ (buffered ||= []) << server if sock.respond_to?(:pending_data?) && sock.pending_data?
257
+ end
258
+ return buffered if buffered
259
+
250
260
  sockets = servers.map(&:sock)
251
261
  readable, = IO.select(sockets, nil, nil, timeout)
252
262
  return [] if readable.nil?
@@ -185,6 +185,9 @@ module Dalli
185
185
  # to the noop at the end of the pipeline
186
186
  finish_pipeline && break if status && key.nil?
187
187
 
188
+ @pipeline_response_count += 1
189
+ raise Dalli::DalliError, 'Too many get_multi replies' if @pipeline_response_count > @expected_response_count
190
+
188
191
  # A key that wasn't requested means the replies are out of step
189
192
  # (see expect_reply_keys): keep what was read so far, drop the rest
190
193
  if unexpected_reply_key?(key)
@@ -495,6 +498,9 @@ module Dalli
495
498
  status, cas, key, value = response_buffer.process_single_getk_response
496
499
  break if status.nil? # No complete response available
497
500
 
501
+ @pipeline_response_count += 1
502
+ raise Dalli::DalliError, 'Too many get_multi replies' if @pipeline_response_count > @expected_response_count
503
+
498
504
  # Handled by the next pipeline_next_responses, once sending is done
499
505
  if unexpected_reply_key?(key)
500
506
  @pipeline_out_of_step = true
@@ -528,6 +534,8 @@ module Dalli
528
534
  # other clients created, on memcached versions with the behavior.
529
535
  def expect_reply_keys(keys)
530
536
  @pipeline_out_of_step = false
537
+ @expected_response_count = keys.size
538
+ @pipeline_response_count = 0
531
539
  @expected_reply_keys = (keys.to_set { |k| reply_key_form(k) } if @verify_reply_keys)
532
540
  end
533
541
 
@@ -578,6 +586,8 @@ module Dalli
578
586
  response_buffer.clear
579
587
  @pipeline_out_of_step = false
580
588
  @expected_reply_keys = nil
589
+ @expected_response_count = nil
590
+ @pipeline_response_count = nil
581
591
  end
582
592
 
583
593
  # The single-server set_multi path passes its whole pairs hash as the
@@ -267,7 +267,7 @@ module Dalli
267
267
 
268
268
  def read_line
269
269
  flush_write_buffer
270
- data = @sock.gets("\r\n", MAX_LINE_BYTES)
270
+ data = timed_tls_operation { @sock.gets("\r\n", MAX_LINE_BYTES) }
271
271
  error_on_request!('EOF in read_line') if data.nil?
272
272
  # Without its CRLF the line either ran past MAX_LINE_BYTES or was cut
273
273
  # short by EOF; either way the connection is out of step
@@ -434,6 +434,16 @@ module Dalli
434
434
 
435
435
  private
436
436
 
437
+ # Ruby 3.3's default OpenSSL can block past IO#timeout= on the TCP
438
+ # socket. Bound the blocking TLS call itself; error_on_request! closes
439
+ # the connection after the Timeout::Error, so a partial reply or write
440
+ # is never reused.
441
+ def timed_tls_operation(&)
442
+ return yield unless @sock.is_a?(Dalli::Socket::SSLSocket)
443
+
444
+ Timeout.timeout(socket_timeout, &)
445
+ end
446
+
437
447
  # One write per flush. A TLS socket sends each argument of
438
448
  # write(*parts) as its own record and system call, so the parts are
439
449
  # joined first.
@@ -442,7 +452,7 @@ module Dalli
442
452
 
443
453
  # Set before writing: a write that fails may still have sent some bytes
444
454
  @request_sent = true if @request_in_progress
445
- @sock.write(@write_buffer.size == 1 ? @write_buffer.first : joined_write_buffer)
455
+ timed_tls_operation { @sock.write(@write_buffer.size == 1 ? @write_buffer.first : joined_write_buffer) }
446
456
  discard_write_buffer
447
457
  end
448
458
 
@@ -475,7 +485,7 @@ module Dalli
475
485
  # error_on_request!, which closes the dirty socket for a retry on a fresh
476
486
  # connection (and preserves the $ERROR_INFO context down! relies on).
477
487
  def read_bytes(count)
478
- buffer = @sock.read(count)
488
+ buffer = timed_tls_operation { @sock.read(count) }
479
489
  return buffer if buffer && buffer.bytesize == count
480
490
 
481
491
  raise EOFError, "EOF reading #{count} bytes; received #{buffer ? buffer.bytesize : 0}"
@@ -238,7 +238,7 @@ module Dalli
238
238
 
239
239
  def finish_pipelined_delete(sent)
240
240
  write_noop
241
- sent - response_processor.pipelined_delete_non_deletions
241
+ sent - response_processor.pipelined_delete_non_deletions(sent)
242
242
  end
243
243
 
244
244
  # Arithmetic Commands
@@ -373,8 +373,13 @@ module Dalli
373
373
  expect_reply_keys(keys)
374
374
  hash = {}
375
375
  unmarshal_error = nil
376
+ replies = 0
376
377
  while (line = @connection_manager.read_line)
377
378
  break if line.start_with?('MN')
379
+
380
+ replies += 1
381
+ raise Dalli::DalliError, 'Too many get_multi replies' if replies > keys.size
382
+
378
383
  next unless line.start_with?('VA ')
379
384
 
380
385
  tokens = line.chomp!(TERMINATOR).split
@@ -408,8 +413,13 @@ module Dalli
408
413
  expect_reply_keys(keys)
409
414
  hash = {}
410
415
  unmarshal_error = nil
416
+ replies = 0
411
417
  while (line = @connection_manager.read_line)
412
418
  break if line.start_with?('MN')
419
+
420
+ replies += 1
421
+ raise Dalli::DalliError, 'Too many get_multi replies' if replies > keys.size
422
+
413
423
  next unless line.start_with?('VA ')
414
424
 
415
425
  begin
@@ -467,7 +477,7 @@ module Dalli
467
477
  buffer = RequestFormatter.multi_meta_set(entries, ttl: ttl, **routing_token_kwargs(req_options))
468
478
  flushed_write(buffer)
469
479
  buffer.clear
470
- response_processor.consume_all_responses_until_mn
480
+ response_processor.consume_all_responses_until_mn(entries.size)
471
481
  end
472
482
  raise value_error if value_error
473
483
  end
@@ -479,7 +489,7 @@ module Dalli
479
489
  **routing_token_kwargs(req_options))
480
490
  flushed_write(buffer)
481
491
  buffer.clear
482
- keys.size - response_processor.pipelined_delete_non_deletions
492
+ keys.size - response_processor.pipelined_delete_non_deletions(keys.size)
483
493
  end
484
494
 
485
495
  require_relative 'key_regularizer'
@@ -26,6 +26,8 @@ module Dalli
26
26
  VA_PREFIX = 'VA '
27
27
  # memcached can't store an item larger than 1 GiB (its -I maximum)
28
28
  MAX_VALUE_BYTES = 1024 * 1024 * 1024
29
+ MAX_STATS_ENTRIES = 10_000
30
+ MAX_QUIET_RESPONSES = 100_000
29
31
  HD_PREFIX = 'HD '
30
32
  FLAGS_TOKEN_PREFIX = ' f'
31
33
  CAS_TOKEN_PREFIX = ' c'
@@ -176,7 +178,11 @@ module Dalli
176
178
  def stats
177
179
  tokens = error_on_unexpected!(T_END_TOKEN_STAT)
178
180
  values = {}
181
+ replies = 0
179
182
  while tokens.first != END_TOKEN
183
+ replies += 1
184
+ raise Dalli::DalliError, 'Too many stats replies' if replies > MAX_STATS_ENTRIES
185
+
180
186
  values[tokens[1]] = tokens[2]
181
187
  tokens = next_line_to_tokens
182
188
  end
@@ -200,10 +206,16 @@ module Dalli
200
206
  tokens.last
201
207
  end
202
208
 
203
- def consume_all_responses_until_mn
209
+ def consume_all_responses_until_mn(max_responses = MAX_QUIET_RESPONSES)
204
210
  tokens = next_line_to_tokens
205
211
 
206
- tokens = next_line_to_tokens while tokens.first != MN
212
+ replies = 0
213
+ while tokens.first != MN
214
+ replies += 1
215
+ raise Dalli::DalliError, 'Too many quiet replies' if replies > max_responses
216
+
217
+ tokens = next_line_to_tokens
218
+ end
207
219
  true
208
220
  end
209
221
 
@@ -214,11 +226,13 @@ module Dalli
214
226
  # was NOT deleted -- a miss (NF) or an error. Returns that count so
215
227
  # callers can derive the number of successful deletes as
216
228
  # (keys_sent - non_deletions).
217
- def pipelined_delete_non_deletions
229
+ def pipelined_delete_non_deletions(max_responses = MAX_QUIET_RESPONSES)
218
230
  non_deletions = 0
219
231
  tokens = next_line_to_tokens
220
232
  until tokens.first == MN
221
233
  non_deletions += 1
234
+ raise Dalli::DalliError, 'Too many delete replies' if non_deletions > max_responses
235
+
222
236
  tokens = next_line_to_tokens
223
237
  end
224
238
  non_deletions
@@ -239,6 +253,13 @@ module Dalli
239
253
  def getk_response_from_buffer(buf, offset = 0, raw: false)
240
254
  # Find the header terminator starting from offset
241
255
  term_idx = buf.byteindex(TERMINATOR, offset)
256
+ max_line_bytes = ConnectionManager::MAX_LINE_BYTES
257
+ header_too_long = if term_idx
258
+ term_idx - offset + TERMINATOR.bytesize > max_line_bytes
259
+ else
260
+ buf.bytesize - offset >= max_line_bytes
261
+ end
262
+ raise Dalli::DalliError, "Reply line without CRLF in the first #{max_line_bytes} bytes" if header_too_long
242
263
  return [0] unless term_idx
243
264
 
244
265
  if buf.byteslice(offset, VA_PREFIX.bytesize) == VA_PREFIX
@@ -488,9 +509,11 @@ module Dalli
488
509
  end
489
510
 
490
511
  def check_value_size!(size)
491
- return if size.between?(0, MAX_VALUE_BYTES)
512
+ limit = @value_marshaller.max_response_bytes
513
+ return if size >= 0 && size <= MAX_VALUE_BYTES && size <= limit # rubocop:disable Style/ComparableBetween
492
514
 
493
- raise Dalli::DalliError, "Reply value size #{size} is out of range"
515
+ effective_limit = [MAX_VALUE_BYTES, limit].min
516
+ raise Dalli::DalliError, "Reply value size #{size} exceeds the #{effective_limit}-byte response limit"
494
517
  end
495
518
 
496
519
  # A value is read by its declared size, so check the CRLF that should
@@ -15,12 +15,14 @@ module Dalli
15
15
  value_max_bytes: 1024 * 1024
16
16
  }.freeze
17
17
 
18
- attr_reader :value_max_bytes
18
+ attr_reader :value_max_bytes, :max_response_bytes
19
19
 
20
20
  def initialize(client_options)
21
21
  @value_max_bytes = client_options.fetch(:value_max_bytes) do
22
22
  DEFAULTS.fetch(:value_max_bytes)
23
23
  end.to_i
24
+ @max_response_bytes = Integer(client_options.fetch(:max_response_bytes, @value_max_bytes))
25
+ raise ArgumentError, 'max_response_bytes must be positive' unless @max_response_bytes.positive?
24
26
  end
25
27
 
26
28
  def store(key, value, _options = nil)
@@ -47,6 +47,8 @@ module Dalli
47
47
  @marshal_options =
48
48
  DEFAULTS.merge(client_options.slice(*OPTIONS))
49
49
  @marshal_options[:value_max_bytes] = @marshal_options[:value_max_bytes].to_i
50
+ @max_response_bytes = Integer(client_options.fetch(:max_response_bytes, value_max_bytes))
51
+ raise ArgumentError, 'max_response_bytes must be positive' unless @max_response_bytes.positive?
50
52
  end
51
53
 
52
54
  def store(key, value, options = nil)
@@ -67,6 +69,8 @@ module Dalli
67
69
  @marshal_options[:value_max_bytes]
68
70
  end
69
71
 
72
+ attr_reader :max_response_bytes
73
+
70
74
  def error_if_over_max_value_bytes(key, value)
71
75
  self.class.error_if_over_max_value_bytes(key, value, value_max_bytes)
72
76
  end
data/lib/dalli/socket.rb CHANGED
@@ -19,6 +19,10 @@ module Dalli
19
19
  ##
20
20
  module InstanceMethods
21
21
  READ_CHUNK_SIZE = 8196
22
+ # Give the response parser control between bounded socket reads. In
23
+ # particular, a peer continuously sending a header without CRLF must
24
+ # not keep this method appending forever.
25
+ MAX_AVAILABLE_BYTES = READ_CHUNK_SIZE * 8
22
26
 
23
27
  # Reads whatever is available without blocking. A read that returns less
24
28
  # than a full chunk means the socket was drained, so we stop there rather
@@ -26,7 +30,7 @@ module Dalli
26
30
  # Callers keep waiting on IO.select for anything that arrives later.
27
31
  def read_available(reusable_buffer = nil)
28
32
  if reusable_buffer
29
- value = read_nonblock(READ_CHUNK_SIZE, reusable_buffer, exception: false)
33
+ value = read_nonblock([READ_CHUNK_SIZE, MAX_AVAILABLE_BYTES].min, reusable_buffer, exception: false)
30
34
  case value
31
35
  when :wait_writable, :wait_readable
32
36
  return reusable_buffer.clear
@@ -34,13 +38,15 @@ module Dalli
34
38
  raise Errno::ECONNRESET, "Connection reset: #{logged_options.inspect}"
35
39
  end
36
40
  return value if drained_after?(value)
41
+ return value if value.bytesize >= MAX_AVAILABLE_BYTES
37
42
  else
38
43
  value = ''.b
39
44
  end
40
45
 
41
46
  buffer = ''.b
42
47
  loop do
43
- result = read_nonblock(READ_CHUNK_SIZE, buffer, exception: false)
48
+ result = read_nonblock([READ_CHUNK_SIZE, MAX_AVAILABLE_BYTES - value.bytesize].min,
49
+ buffer, exception: false)
44
50
  case result
45
51
  when :wait_writable, :wait_readable
46
52
  buffer.clear
@@ -50,6 +56,7 @@ module Dalli
50
56
  else
51
57
  value << result
52
58
  return value if drained_after?(result)
59
+ return value if value.bytesize >= MAX_AVAILABLE_BYTES
53
60
  end
54
61
  end
55
62
  end
@@ -114,13 +121,17 @@ module Dalli
114
121
  # jruby-openssl can take the next record off the socket during a short
115
122
  # read and hold it where neither pending nor IO.select sees it, so on
116
123
  # JRuby a read continues until the socket would block.
124
+ def pending_data?
125
+ pending.positive? || (@rbuffer && !@rbuffer.empty?)
126
+ end
127
+
117
128
  if RUBY_ENGINE == 'jruby'
118
129
  def buffered_data?
119
130
  true
120
131
  end
121
132
  else
122
133
  def buffered_data?
123
- pending.positive?
134
+ pending_data?
124
135
  end
125
136
  end
126
137
 
@@ -157,7 +168,12 @@ module Dalli
157
168
  sock.options = { host: host, port: port }.merge(options)
158
169
  init_socket_options(sock, options)
159
170
 
160
- options[:ssl_context] ? wrapping_ssl_socket(sock, host, options[:ssl_context]) : sock
171
+ if options[:ssl_context]
172
+ wrapping_ssl_socket(sock, host, options[:ssl_context],
173
+ options[:socket_timeout])
174
+ else
175
+ sock
176
+ end
161
177
  end
162
178
  end
163
179
 
@@ -251,13 +267,15 @@ module Dalli
251
267
 
252
268
  # Closes the TCP socket if the handshake fails or times out, so a failed
253
269
  # attempt doesn't hold a file descriptor until GC.
254
- def self.wrapping_ssl_socket(tcp_socket, host, ssl_context)
270
+ def self.wrapping_ssl_socket(tcp_socket, host, ssl_context, timeout = nil)
255
271
  warn_if_unverified(ssl_context)
256
272
  connected = false
257
273
  ssl_socket = Dalli::Socket::SSLSocket.new(tcp_socket, ssl_context)
258
274
  ssl_socket.hostname = host
259
275
  ssl_socket.sync_close = true
260
- ssl_socket.connect
276
+ # The underlying TCP socket's IO#timeout= does not reliably bound
277
+ # OpenSSL's blocking handshake on Ruby 3.3.
278
+ Timeout.timeout(timeout) { ssl_socket.connect }
261
279
  connected = true
262
280
  ssl_socket
263
281
  ensure
data/lib/dalli/version.rb CHANGED
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Dalli
4
- VERSION = '5.3.2'
4
+ VERSION = '5.3.3'
5
5
 
6
6
  MIN_SUPPORTED_MEMCACHED_VERSION = '1.6.27'
7
7
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dalli
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.3.2
4
+ version: 5.3.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Peter M. Goldstein