dalli 5.1.3 → 5.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf1d386c0e4bbde88c4dcafcd1babe51ffa12eaa173e105019bdcce2f7a2acfa
-  data.tar.gz: 07463ea0880be455439262d77dc8255562a0ed7035e5ec29ee07888612d844a2
+  metadata.gz: 70f17667b9c623f9233be342d920de385bab797b44a4bc9f1e85c233f766404f
+  data.tar.gz: 527b9ff3dd4699ad5d307107668b31d3a739675825ad4d0b2cfb35ac09581a81
 SHA512:
-  metadata.gz: 98f088e6d7c01b13b912a18f4c870587d02db32ebb3b6ec471359f97bf15324261468b7d62186f1ffc03bfe66ff175884c18f551fd457cf6a80dc2aef1774608
-  data.tar.gz: b3919430e8b9ed967e52495d626013b54a4c8a8b62c8a3042224569d6594b455188ff5d0ceb2342810994e5dc419dea780bf969c9cfa918395387127758a1b96
+  metadata.gz: 9e54f454a1bca3b79f229a219b458533d2778c1521043c1ee7c2d203ae0d25e1449fc375b59754aaadac79de6ea484170ba52c10008da6abd40451821300a60c
+  data.tar.gz: 1dea61992e48d8786dbb3ef929e30c53d3544bafed895334cf4316ad0710791c4ebb0887fb4fa63b79358c0029be11b3495a2875a24956a296246fe9d5e4eaa0

data/CHANGELOG.md

@@ -4,6 +4,32 @@ Dalli Changelog
 Unreleased
 ==========
 
+5.1.4
+==========
+
+Security:
+
+- Keep the caller's key when retrying a request (GHSA-m252-9cgf-vx2w)
+  - With a `namespace`, a request retried after a transient network error (a timeout, or a connection closed by memcached or a proxy) applied the namespace a second time, so a retried read could return a different key's value and a retried write could overwrite a different key
+  - Affects single-key operations since 5.0.3, single-server `get_multi` since 5.1.0, and `get_with_metadata` and `fetch_with_lock` since 4.1.0; fixed in 5.2.2, 5.1.4, 5.0.10 and 4.3.7. Clients without a namespace aren't affected
+- Complete the fix for per-request `raw: true` on reads (GHSA-wr87-m4jw-29x5)
+  - A read made with `raw: true` doesn't ask for flags, but a reply carrying them anyway (from a proxy or a hostile server) still had its value deserialized. Raw reads now ignore flags in the reply
+  - Affects 5.1.3 and earlier; fixed in 5.2.2, 5.1.4, 5.0.10, 4.3.7 and 3.2.13
+
+Bug fixes:
+
+- `get_with_metadata` and `fetch_with_lock` retried the final error raised when a server is marked down, so with `down_retry_delay: 0` they retried an unresponsive server forever. They now retry only retryable errors, like other operations
+- Fix a regression in 5.1.3: a `get_multi` that didn't finish within `socket_timeout` counted toward `socket_max_failures`, so two slow `get_multi` calls in a row marked a healthy server down. It now just closes the connection
+- Fix a regression in 5.1.3: over TLS, every request buffered in a `quiet` block was sent as its own TLS record and system call (2.5 times slower for a block of 2000 deletes). Each flush is one write again
+- Handle malformed replies from a broken or hostile server or proxy: a negative value size, a hit with no key, or a hit with no `s` flag could leave the connection out of step or raise a non-Dalli error
+- With `decompressed_max_bytes` in effect, data after the end of a compressed value's stream was returned as part of the value. It's now ignored, as without the limit
+- With a `digest_class` whose digests aren't short hex strings, shortening a long key could loop forever. It now raises `ArgumentError`
+
+Development:
+
+- Run the Tests, RuboCop and Profiles workflows on pushes to `main` and the `*-stable` branches only, so a pull request's branch isn't tested twice (backport of #1198)
+- Fix flaky failover tests: move their ports out of Linux's ephemeral port range, and wait for memcached to accept connections after starting it (backport of #1184)
+
 5.1.3
 ==========
 

data/lib/dalli/client.rb

@@ -136,14 +136,18 @@ module Dalli
     #
     def get_with_metadata(key, options = {})
       validate_routing_tokens!(options)
-      key = key.to_s
-      key = @key_manager.validate_key(key)
+      # A retry re-runs this method with its original arguments, so `key` must
+      # stay as the caller gave it; reassigning it would add the namespace again.
+      validated_key = @key_manager.validate_key(key.to_s)
 
-      server = ring.server_for_key(key)
-      Instrumentation.trace('get_with_metadata', trace_attrs('get_with_metadata', key, server)) do
-        server.request(:meta_get, key, options)
+      server = ring.server_for_key(validated_key)
+      Instrumentation.trace('get_with_metadata', trace_attrs('get_with_metadata', validated_key, server)) do
+        server.request(:meta_get, validated_key, options)
       end
-    rescue NetworkError => e
+    # Only a retryable error is retried. The NetworkError raised when a server
+    # is marked down is final, as in #perform; retrying it looped forever once
+    # down_retry_delay let the same server be tried again.
+    rescue RetryableNetworkError => e
       Dalli.logger.debug { e.inspect }
       Dalli.logger.debug { 'retrying get_with_metadata with new server' }
       retry
@@ -302,14 +306,17 @@ module Dalli
       validate_integer!(:lock_ttl, lock_ttl)
       validate_integer!(:recache_threshold, recache_threshold)
       validate_routing_tokens!(req_options)
-      key = key.to_s
-      key = @key_manager.validate_key(key)
+      # Left unchanged for the retry below; see get_with_metadata
+      validated_key = @key_manager.validate_key(key.to_s)
 
-      server = ring.server_for_key(key)
-      Instrumentation.trace('fetch_with_lock', trace_attrs('fetch_with_lock', key, server)) do
-        fetch_with_lock_request(key, ttl, lock_ttl, recache_threshold, req_options, &block)
+      server = ring.server_for_key(validated_key)
+      Instrumentation.trace('fetch_with_lock', trace_attrs('fetch_with_lock', validated_key, server)) do
+        fetch_with_lock_request(validated_key, ttl, lock_ttl, recache_threshold, req_options, &block)
       end
-    rescue NetworkError => e
+    # Only a retryable error is retried. The NetworkError raised when a server
+    # is marked down is final, as in #perform; retrying it looped forever once
+    # down_retry_delay let the same server be tried again.
+    rescue RetryableNetworkError => e
       Dalli.logger.debug { e.inspect }
       Dalli.logger.debug { 'retrying fetch_with_lock with new server' }
       retry
@@ -727,10 +734,11 @@ module Dalli
     # to route to at all is still silent, matching Ring#keys_grouped_by_server
     # dropping a key it can't route on both the single- and multi-server paths.
     def single_server_get_multi(keys, req_options = nil)
-      keys.map! { |k| @key_manager.validate_key(k.to_s) }
+      # A new array, so the retry below starts again from the caller's keys
+      validated_keys = keys.map { |k| @key_manager.validate_key(k.to_s) }
       return {} unless (server = single_server)
 
-      result = server.request(:read_multi_req, keys, req_options)
+      result = server.request(:read_multi_req, validated_keys, req_options)
       result.transform_keys! { |k| @key_manager.key_without_namespace(k) }
       result
     rescue Dalli::RetryableNetworkError => e
@@ -897,18 +905,20 @@ module Dalli
       # rubocop:enable Naming/MethodParameterName
       return yield if block_given?
 
-      key = key.to_s
-      key = @key_manager.validate_key(key)
+      # `retry` below re-runs this method with its original arguments, so `key`
+      # must stay as the caller gave it: reassigning it would apply the
+      # namespace a second time and send the retry to a different key.
+      validated_key = @key_manager.validate_key(key.to_s)
 
-      server = ring.server_for_key(key)
+      server = ring.server_for_key(validated_key)
 
       if Instrumentation.enabled?
         op_name = op.name
-        Instrumentation.trace(op_name, trace_attrs(op_name, key, server)) do
-          server.request(op, key, ...)
+        Instrumentation.trace(op_name, trace_attrs(op_name, validated_key, server)) do
+          server.request(op, validated_key, ...)
         end
       else
-        server.request(op, key, ...)
+        server.request(op, validated_key, ...)
       end
     rescue RetryableNetworkError => e
       Dalli.logger.debug { e.inspect }

data/lib/dalli/compressor.rb

@@ -39,7 +39,12 @@ module Dalli
       out = String.new(encoding: Encoding::BINARY)
       0.step(data.bytesize - 1, INFLATE_SLICE_BYTES) do |pos|
         inflate_slice(inflater, data.byteslice(pos, INFLATE_SLICE_BYTES), out, max_bytes)
+        # Anything after the end of the stream isn't part of the value, as
+        # with Zlib::Inflate.inflate; finish would return it as output
+        break if inflater.finished?
       end
+      return out if inflater.finished?
+
       append_within_limit(out, inflater.finish, max_bytes)
     ensure
       inflater.close

data/lib/dalli/key_manager.rb

@@ -122,15 +122,18 @@ module Dalli
     ##
     def truncated_key(key)
       digest = digest_class.hexdigest(key)
-      prefix = key[0, prefix_length(digest)]
+      prefix = key[0, [prefix_length(digest), 0].max]
       truncated = "#{prefix}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
       # A prefix with whitespace or non-ASCII characters sends the whole key
       # base64-encoded, which can still be too long on the wire; shorten it.
-      while wire_length(truncated) > MAX_KEY_LENGTH
+      while wire_length(truncated) > MAX_KEY_LENGTH && !prefix.empty?
         prefix = prefix[0...-1]
         truncated = "#{prefix}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
       end
-      truncated
+      return truncated if wire_length(truncated) <= MAX_KEY_LENGTH
+
+      # Only possible with a digest_class whose digests aren't short hex strings
+      raise ArgumentError, "#{digest_class} digests are too long to fit in a truncated key"
     end
 
     # memcached's key limit applies to the key as sent: in bytes, and after

data/lib/dalli/protocol/base.rb

@@ -197,10 +197,13 @@ module Dalli
         @connection_manager.abort_request!
         return true unless connected?
 
-        # Closes the connection, which ensures that our connection
-        # is in a clean state for future requests
-        @connection_manager.error_on_request!('External timeout')
-      rescue NetworkError
+        # Closes the connection, which ensures that our connection is in a
+        # clean state for future requests. The get_multi ran out of time
+        # overall rather than hitting a socket error, so it doesn't count
+        # toward socket_max_failures: two slow get_multis in a row would
+        # otherwise mark a healthy server down.
+        Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
+        @connection_manager.close
         true
       end
 
@@ -328,6 +331,7 @@ module Dalli
         # Use clear (not reset) to keep pipeline_complete? = true, which is
         # the expected state before pipeline_response_setup is called.
         response_buffer.clear
+        response_buffer.raw = raw_request?(options)
 
         # The terminating noop is sent by pipeline_response_setup
         write(quiet_get_requests(keys, options, return_cas: return_cas))
@@ -339,6 +343,7 @@ module Dalli
       def pipelined_get_interleaved(keys, chunk_size, results, options = nil)
         # Initialize the response buffer for draining during send phase
         response_buffer.ensure_ready
+        response_buffer.raw = raw_request?(options)
 
         keys.each_slice(chunk_size) do |chunk|
           # Build and write this chunk of requests

data/lib/dalli/protocol/connection_manager.rb

@@ -334,13 +334,32 @@ module Dalli
 
       private
 
+      # One write per flush. A TLS socket sends each argument of
+      # write(*parts) as its own record and system call, so the parts are
+      # joined first.
       def flush_write_buffer
         return if @write_buffer.empty?
 
-        @sock.write(*@write_buffer)
+        @sock.write(@write_buffer.size == 1 ? @write_buffer.first : joined_write_buffer)
         discard_write_buffer
       end
 
+      # Joined as bytes: requests can mix UTF-8 and binary strings, which
+      # String#<< refuses to combine when both hold non-ASCII bytes
+      if String.method_defined?(:append_as_bytes)
+        def joined_write_buffer
+          out = String.new(capacity: @write_buffer_bytes, encoding: Encoding::BINARY)
+          @write_buffer.each { |part| out.append_as_bytes(part) }
+          out
+        end
+      else
+        def joined_write_buffer
+          out = String.new(capacity: @write_buffer_bytes, encoding: Encoding::BINARY)
+          @write_buffer.each { |part| out << part.b }
+          out
+        end
+      end
+
       def discard_write_buffer
         @write_buffer.clear
         @write_buffer_bytes = 0

data/lib/dalli/protocol/meta.rb

@@ -38,7 +38,7 @@ module Dalli
                 RequestFormatter.plain_meta_get(key, skip_flags)
               end
         flushed_write(req)
-        response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
+        response_processor.meta_get_with_value(cache_nils: cache_nils?(options), raw: raw_request?(options))
       end
 
       def quiet_get_request(key, options = nil)
@@ -59,7 +59,7 @@ module Dalli
         skip_flags = raw_mode? || (options && options[:raw])
         req = RequestFormatter.meta_get(key: key, ttl: ttl, skip_flags: skip_flags, **routing_token_kwargs(options))
         flushed_write(req)
-        response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
+        response_processor.meta_get_with_value(cache_nils: cache_nils?(options), raw: raw_request?(options))
       end
 
       def touch(key, ttl)
@@ -75,7 +75,7 @@ module Dalli
         req = RequestFormatter.meta_get(key: key, value: true, return_cas: true, skip_flags: raw_request?(options),
                                         **routing_token_kwargs(options))
         flushed_write(req)
-        response_processor.meta_get_with_value_and_cas
+        response_processor.meta_get_with_value_and_cas(raw: raw_request?(options))
       end
 
       # Comprehensive meta get with support for all metadata flags.
@@ -116,7 +116,7 @@ module Dalli
         response_processor.meta_get_with_metadata(
           cache_nils: cache_nils?(options), return_hit_status: options[:return_hit_status],
           return_last_access: options[:return_last_access],
-          return_ttl_remaining: options[:return_ttl_remaining]
+          return_ttl_remaining: options[:return_ttl_remaining], raw: raw_request?(options)
         )
       end
 
@@ -300,7 +300,9 @@ module Dalli
           next unless line.start_with?('VA ')
 
           tokens = line.chomp!(TERMINATOR).split
-          value = @connection_manager.read(tokens[1].to_i + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
+          size = tokens[1].to_i
+          response_processor.check_value_size!(size)
+          value = @connection_manager.read(size + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
           stale = response_processor.stale_from_tokens(tokens)
           cas = response_processor.cas_from_tokens(tokens)
           bitflags = is_raw ? 0 : response_processor.bitflags_from_tokens(tokens)
@@ -327,7 +329,9 @@ module Dalli
 
       def parse_multi_get_value(line, key_index, is_raw)
         tokens = line.chomp!(TERMINATOR).split
-        value = @connection_manager.read(tokens[1].to_i + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
+        size = tokens[1].to_i
+        response_processor.check_value_size!(size)
+        value = @connection_manager.read(size + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
         raw_key = tokens[key_index]
         return unless raw_key
 

data/lib/dalli/protocol/response_buffer.rb

@@ -15,6 +15,9 @@ module Dalli
     # when advancing through parsed responses.
     ##
     class ResponseBuffer
+      # Whether the current pipeline was requested with raw: true
+      attr_writer :raw
+
       def initialize(io_source, response_processor)
         @io_source = io_source
         @response_processor = response_processor
@@ -39,7 +42,7 @@ module Dalli
       # Attempts to process a single response from the buffer,
       # advancing the offset past the consumed bytes.
       def process_single_getk_response
-        response = @response_processor.getk_response_from_buffer(@buffer, @offset)
+        response = @response_processor.getk_response_from_buffer(@buffer, @offset, raw: @raw || false)
         @offset += response.pop
         response
       end

data/lib/dalli/protocol/response_processor.rb

@@ -48,12 +48,15 @@ module Dalli
           @value_marshaller = value_marshaller
         end
 
-        def meta_get_with_value(cache_nils: false)
+        # raw: true (a request made with raw: true) returns the value as stored,
+        # ignoring any flags in the reply. The request didn't ask for flags, so
+        # a reply carrying them anyway must not get the value deserialized.
+        def meta_get_with_value(cache_nils: false, raw: false)
           line = read_line
           # A hit ("VA <size> f<flags>") is parsed in place rather than split
           # into tokens, which saves several allocations on the hottest path.
           if line&.start_with?(VA_PREFIX)
-            return @value_marshaller.retrieve(read_data(size_from_va_line(line)), bitflags_from_va_line(line))
+            return retrieve(read_data(size_from_va_line(line)), raw ? 0 : bitflags_from_va_line(line), raw)
           end
 
           tokens = line&.split || []
@@ -61,7 +64,7 @@ module Dalli
           when EN
             cache_nils ? ::Dalli::NOT_FOUND : nil
           when VA # only reached for an unusually formatted hit line
-            @value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
+            retrieve(read_data(tokens[1].to_i), raw ? 0 : bitflags_from_tokens(tokens), raw)
           when HD
             true
           else
@@ -69,14 +72,14 @@ module Dalli
           end
         end
 
-        def meta_get_with_value_and_cas
+        def meta_get_with_value_and_cas(raw: false)
           tokens = error_on_unexpected!(T_VA_EN_HD)
           return [nil, 0] if tokens.first == EN
 
           cas = cas_from_tokens(tokens)
           return [nil, cas] unless tokens.first == VA
 
-          [@value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens)), cas]
+          [retrieve(read_data(tokens[1].to_i), raw ? 0 : bitflags_from_tokens(tokens), raw), cas]
         end
 
         def meta_get_without_value
@@ -96,13 +99,13 @@ module Dalli
         # Used by meta_get for comprehensive metadata retrieval.
         # Supports thundering herd protection (N/R flags) and metadata flags (h/l/u).
         def meta_get_with_metadata(cache_nils: false, return_hit_status: false, return_last_access: false,
-                                   return_ttl_remaining: false)
+                                   return_ttl_remaining: false, raw: false)
           tokens = error_on_unexpected!(T_VA_EN_HD)
           result = build_metadata_result(tokens)
           result[:hit_before] = hit_status_from_tokens(tokens) if return_hit_status
           result[:last_access] = last_access_from_tokens(tokens) if return_last_access
           result[:ttl_remaining] = ttl_remaining_from_tokens(tokens) if return_ttl_remaining
-          result[:value] = parse_value_from_tokens(tokens, cache_nils)
+          result[:value] = parse_value_from_tokens(tokens, cache_nils, raw)
           result
         end
 
@@ -119,11 +122,11 @@ module Dalli
           }
         end
 
-        def parse_value_from_tokens(tokens, cache_nils)
+        def parse_value_from_tokens(tokens, cache_nils, raw = false) # rubocop:disable Style/OptionalBooleanParameter
           return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
           return unless tokens.first == VA
 
-          @value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
+          retrieve(read_data(tokens[1].to_i), raw ? 0 : bitflags_from_tokens(tokens), raw)
         end
 
         def meta_set_with_cas
@@ -214,7 +217,9 @@ module Dalli
         # The remaining three values in the array are the ResponseHeader,
         # key, and value.
         ##
-        def getk_response_from_buffer(buf, offset = 0)
+        # raw: true returns values as stored, ignoring flags in the replies;
+        # see meta_get_with_value.
+        def getk_response_from_buffer(buf, offset = 0, raw: false)
           # Find the header terminator starting from offset
           term_idx = buf.byteindex(TERMINATOR, offset)
           return [0] unless term_idx
@@ -236,7 +241,9 @@ module Dalli
             when BYTE_B then base64 ||= token == 'b'
             end
           end
-          body_len = flag_int(size, 's')
+          # memcached also gives the size positionally ("VA <size> ..."); use it
+          # when there's no s flag rather than reading the value as replies
+          body_len = size ? flag_int(size, 's') : positional_size(tokens)
 
           # A complete response with no body. Only the terminating noop's MN
           # ends the pipeline. Any other bodyless reply (CLIENT_ERROR,
@@ -256,12 +263,21 @@ module Dalli
           # The full response is in our buffer, so parse it and return
           # the values
           body = buf.byteslice(offset + header_len, body_len)
-          value = @value_marshaller.retrieve(body, flag_int(bitflags, 'f'))
-          key = key ? key.delete_prefix!('k') : 0
+          value = retrieve(body, raw ? 0 : flag_int(bitflags, 'f'), raw)
+          # A hit with no key can't be matched to a request: skip it (false
+          # status) rather than yield it, or take it for the end of the pipeline
+          return [false, resp_size] unless key
+
+          key = key.delete_prefix!('k')
           key = KeyRegularizer.decode(key) if base64
           [tokens.first == VA, flag_int(cas, 'c'), key, value, resp_size]
         end
 
+        # Values from a raw request are returned as stored
+        def retrieve(value, bitflags, raw)
+          raw ? value : @value_marshaller.retrieve(value, bitflags)
+        end
+
         # Integer value of a flag token such as "f123", or 0 when absent.
         # Strips the prefix in place, like value_from_tokens.
         def flag_int(token, flag)
@@ -352,6 +368,11 @@ module Dalli
 
         # A pipelined reply claiming an impossible size would otherwise have the
         # buffer wait for (and accumulate) that many bytes
+        # The size given right after "VA", or 0 when it isn't a number
+        def positional_size(tokens)
+          tokens.first == VA && tokens[1]&.match?(/\A\d+\z/) ? tokens[1].to_i : 0
+        end
+
         def check_value_size!(size)
           return if size.between?(0, MAX_VALUE_BYTES)
 
@@ -368,6 +389,9 @@ module Dalli
         end
 
         def read_data(data_size)
+          # Checked before the terminator is added: -1 or -2 would otherwise
+          # read 1 or 0 bytes and leave the connection out of step
+          check_value_size!(data_size)
           @io_source.read(data_size + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
         end
       end

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '5.1.3'
+  VERSION = '5.1.4'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.6.27'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 5.1.3
+  version: 5.1.4
 platform: ruby
 authors:
 - Peter M. Goldstein