dalli 4.3.6 → 4.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd77fe4e8ef65f531c5d9eb4a8e638feb6e443493cd57125bad104867399cd19
-  data.tar.gz: 814cc63401565f306a2aa6045476579f93b028fff8802ea443cde65375328636
+  metadata.gz: 4b1a68fbdf9ea89024d6e9bc5803f94d3342c2f7b8bd7cfea3a97f7a873f7b76
+  data.tar.gz: 93d1c2fcb71fa739e6a00808cddb11567ad9fed3caa4d0751c924576bc04bfe0
 SHA512:
-  metadata.gz: c983d9e656540f313aaea157b8d0290c3610b7c03575625a6e8cfba4ec5be99a893955506b409867689de1a5f4acc7715eee07928df8ef5c8724141ffae243b2
-  data.tar.gz: 62d38b83c8ba58ec871613b25e5f197d14c768436cde9f3aa382189dd6af19802832c545474add6a8b8b9edb373c948cde2db15599b5bba1b15720db71d1cfed
+  metadata.gz: bc42cde8b6f61ba60768bf43a9ebf8b1088dc8b2efbb4e32b5ed1abedd3c3be81c66aa02dc1a2558fd319debcc8074e61471996ad8825afed32d4fd741f8a312
+  data.tar.gz: 3a8320685a39edef196feaa387ef954caea41650b540c7f37d32d4821b5804f41631e5346e14b21912de0963f147c7d55e8c1ad08e24d2f7d25073975ba64d72

data/CHANGELOG.md

@@ -4,6 +4,36 @@ Dalli Changelog
 Unreleased
 ==========
 
+4.3.7
+==========
+
+Security:
+
+- Keep the caller's key when retrying a request (GHSA-m252-9cgf-vx2w)
+  - With a `namespace`, a request retried after a transient network error (a timeout, or a connection closed by memcached or a proxy) applied the namespace a second time, so a retried read could return a different key's value and a retried write could overwrite a different key
+  - Affects `get_with_metadata` and `fetch_with_lock` since 4.1.0 (single-key operations on 4.x aren't affected); fixed in 5.2.2, 5.1.4, 5.0.10 and 4.3.7. Clients without a namespace aren't affected
+- Complete the fix for per-request `raw: true` on reads (GHSA-wr87-m4jw-29x5)
+  - `cas` and `cas!` (both protocols) and `fetch_with_lock` (meta protocol) accepted `raw: true` but didn't pass it to the read, so the value was still deserialized. They now honor it
+  - A read made with `raw: true` doesn't ask for flags, but a reply carrying them anyway (from a proxy or a hostile server) still had its value deserialized. Meta protocol raw reads now ignore flags in the reply
+  - Affects 4.3.6 and earlier; fixed in 5.2.2, 5.1.4, 5.0.10, 4.3.7 and 3.2.13
+- Complete the fork-safety fix (GHSA-w39f-xq2m-4g8x)
+  - Fork detection used a pid cache refreshed by a `Process._fork` hook. When another library's fork hook ran first in the child (connection_pool, loaded before Dalli as in Rails, closes its connections there), Dalli didn't know it was in a forked child and ended the parent's TLS session. It now checks `Process.pid`
+  - 4.3.6's write buffer kept a reference to the caller's value string with the meta protocol, so changing the string before the buffer was sent (for example, reusing one buffer inside a `quiet` block) changed the bytes sent, losing values or adding commands. The buffer now holds a copy
+  - Fixed in 4.3.7 and 3.2.13
+
+Bug fixes:
+
+- `get_with_metadata` and `fetch_with_lock` retried the final error raised when a server is marked down, so with `down_retry_delay: 0` they retried an unresponsive server forever. They now retry only retryable errors, like other operations
+- Fix a regression in 4.3.6: a `get_multi` that didn't finish within `socket_timeout` counted toward `socket_max_failures`, so two slow `get_multi` calls in a row marked a healthy server down. It now just closes the connection
+- Fix a regression in 4.3.6: over TLS, every request buffered in a `quiet` block was sent as its own TLS record and system call (2.5 times slower for a block of 2000 deletes). Each flush is one write again
+- Handle malformed replies from a broken or hostile server or proxy: a negative value size, a hit with no key, or a hit with no `s` flag could leave the connection out of step or raise a non-Dalli error
+- With `decompressed_max_bytes` in effect, data after the end of a compressed value's stream was returned as part of the value. It's now ignored, as without the limit
+- With a `digest_class` whose digests aren't short hex strings, shortening a long key could loop forever. It now raises `ArgumentError`
+
+Development:
+
+- Run the Tests, RuboCop and Profiles workflows on pushes to `main` and the `*-stable` branches only, so a pull request's branch isn't tested twice (backport of #1198)
+
 4.3.6
 ==========
 

data/lib/dalli/client.rb

@@ -134,14 +134,18 @@ module Dalli
     def get_with_metadata(key, options = {})
       raise_unless_meta_protocol!
 
-      key = key.to_s
-      key = @key_manager.validate_key(key)
+      # A retry re-runs this method with its original arguments, so `key` must
+      # stay as the caller gave it; reassigning it would add the namespace again.
+      validated_key = @key_manager.validate_key(key.to_s)
 
-      server = ring.server_for_key(key)
-      Instrumentation.trace('get_with_metadata', trace_attrs('get_with_metadata', key, server)) do
-        server.request(:meta_get, key, options)
+      server = ring.server_for_key(validated_key)
+      Instrumentation.trace('get_with_metadata', trace_attrs('get_with_metadata', validated_key, server)) do
+        server.request(:meta_get, validated_key, options)
       end
-    rescue NetworkError => e
+    # Only a retryable error is retried. The NetworkError raised when a server
+    # is marked down is final, as in #perform; retrying it looped forever once
+    # down_retry_delay let the same server be tried again.
+    rescue RetryableNetworkError => e
       Dalli.logger.debug { e.inspect }
       Dalli.logger.debug { 'retrying get_with_metadata with new server' }
       retry
@@ -238,14 +242,17 @@ module Dalli
       validate_integer!(:lock_ttl, lock_ttl)
       validate_integer!(:recache_threshold, recache_threshold)
 
-      key = key.to_s
-      key = @key_manager.validate_key(key)
+      # Left unchanged for the retry below; see get_with_metadata
+      validated_key = @key_manager.validate_key(key.to_s)
 
-      server = ring.server_for_key(key)
-      Instrumentation.trace('fetch_with_lock', trace_attrs('fetch_with_lock', key, server)) do
-        fetch_with_lock_request(key, ttl, lock_ttl, recache_threshold, req_options, &block)
+      server = ring.server_for_key(validated_key)
+      Instrumentation.trace('fetch_with_lock', trace_attrs('fetch_with_lock', validated_key, server)) do
+        fetch_with_lock_request(validated_key, ttl, lock_ttl, recache_threshold, req_options, &block)
       end
-    rescue NetworkError => e
+    # Only a retryable error is retried. The NetworkError raised when a server
+    # is marked down is final, as in #perform; retrying it looped forever once
+    # down_retry_delay let the same server be tried again.
+    rescue RetryableNetworkError => e
       Dalli.logger.debug { e.inspect }
       Dalli.logger.debug { 'retrying fetch_with_lock with new server' }
       retry
@@ -592,7 +599,8 @@ module Dalli
     end
 
     def cas_core(key, always_set, ttl = nil, req_options = nil)
-      (value, cas) = perform(:cas, key)
+      # req_options reaches the read too, so raw: true returns the stored bytes
+      (value, cas) = perform(:cas, key, req_options)
       return if value.nil? && !always_set
 
       newvalue = yield(value)
@@ -601,7 +609,12 @@ module Dalli
 
     def fetch_with_lock_request(key, ttl, lock_ttl, recache_threshold, req_options)
       server = ring.server_for_key(key)
-      result = server.request(:meta_get, key, { vivify_ttl: lock_ttl, recache_ttl: recache_threshold })
+      # req_options (e.g. raw: true) reaches the read too. It's the base, not
+      # the override: fetch_with_lock's own lock_ttl/recache_threshold win.
+      meta_options = req_options.is_a?(Hash) ? req_options.dup : {}
+      meta_options[:vivify_ttl] = lock_ttl
+      meta_options[:recache_ttl] = recache_threshold
+      result = server.request(:meta_get, key, meta_options)
 
       return result[:value] unless result[:won_recache]
 

data/lib/dalli/compressor.rb

@@ -39,7 +39,12 @@ module Dalli
       out = String.new(encoding: Encoding::BINARY)
       0.step(data.bytesize - 1, INFLATE_SLICE_BYTES) do |pos|
         inflate_slice(inflater, data.byteslice(pos, INFLATE_SLICE_BYTES), out, max_bytes)
+        # Anything after the end of the stream isn't part of the value, as
+        # with Zlib::Inflate.inflate; finish would return it as output
+        break if inflater.finished?
       end
+      return out if inflater.finished?
+
       append_within_limit(out, inflater.finish, max_bytes)
     ensure
       inflater.close

data/lib/dalli/key_manager.rb

@@ -125,15 +125,18 @@ module Dalli
     ##
     def truncated_key(key)
       digest = digest_class.hexdigest(key)
-      prefix = key[0, prefix_length(digest)]
+      prefix = key[0, [prefix_length(digest), 0].max]
       truncated = "#{prefix}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
       # A prefix with whitespace or non-ASCII characters sends the whole key
       # base64-encoded, which can still be too long on the wire; shorten it.
-      while wire_length(truncated) > MAX_KEY_LENGTH
+      while wire_length(truncated) > MAX_KEY_LENGTH && !prefix.empty?
         prefix = prefix[0...-1]
         truncated = "#{prefix}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
       end
-      truncated
+      return truncated if wire_length(truncated) <= MAX_KEY_LENGTH
+
+      # Only possible with a digest_class whose digests aren't short hex strings
+      raise ArgumentError, "#{digest_class} digests are too long to fit in a truncated key"
     end
 
     # memcached's key limit applies to the key as sent: in bytes, and, with

data/lib/dalli/protocol/base.rb

@@ -152,10 +152,13 @@ module Dalli
         @connection_manager.abort_request!
         return true unless connected?
 
-        # Closes the connection, which ensures that our connection
-        # is in a clean state for future requests
-        @connection_manager.error_on_request!('External timeout')
-      rescue NetworkError
+        # Closes the connection, which ensures that our connection is in a
+        # clean state for future requests. The get_multi ran out of time
+        # overall rather than hitting a socket error, so it doesn't count
+        # toward socket_max_failures: two slow get_multis in a row would
+        # otherwise mark a healthy server down.
+        Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
+        @connection_manager.close
         true
       end
 

data/lib/dalli/protocol/binary/response_processor.rb

@@ -102,7 +102,8 @@ module Dalli
           true
         end
 
-        def data_cas_response
+        # raw: true returns the value as stored; see #get
+        def data_cas_response(raw: false)
           resp_header, body = read_response
           return [nil, resp_header.cas] if resp_header.not_found?
           return [nil, false] if resp_header.not_stored?
@@ -110,7 +111,7 @@ module Dalli
           raise_on_not_ok!(resp_header)
           return [nil, resp_header.cas] unless body
 
-          [unpack_response_body(resp_header, body, true).last, resp_header.cas]
+          [unpack_response_body(resp_header, body, !raw).last, resp_header.cas]
         end
 
         # Returns the new value for the key, if found and updated

data/lib/dalli/protocol/binary.rb

@@ -47,11 +47,11 @@ module Dalli
 
       # TODO: This is confusing, as there's a cas command in memcached
       # and this isn't it.  Maybe rename?  Maybe eliminate?
-      def cas(key)
+      def cas(key, options = nil)
         req = RequestFormatter.standard_request(opkey: :get, key: key)
         write(req)
         @connection_manager.flush
-        response_processor.data_cas_response
+        response_processor.data_cas_response(raw: raw_request?(options))
       end
 
       # Storage Commands

data/lib/dalli/protocol/connection_manager.rb

@@ -58,7 +58,7 @@ module Dalli
         @sock = memcached_socket
         # Writes are buffered in @write_buffer instead; see WRITE_BUFFER_FLUSH_BYTES
         @sock.sync = true
-        @pid = PIDCache.pid
+        @pid = Process.pid
         @request_in_progress = false
       rescue SystemCallError, *TIMEOUT_ERRORS, EOFError, SocketError => e
         # SocketError = DNS resolution failure
@@ -204,7 +204,10 @@ module Dalli
       WRITE_BUFFER_FLUSH_BYTES = 64 * 1024
 
       def write(bytes)
-        @write_buffer << bytes
+        # A copy: the caller may change its string before the buffer is sent
+        # (the meta protocol writes a value string as its own part), as Ruby's
+        # IO buffer copied on write
+        @write_buffer << (bytes.frozen? ? bytes : bytes.dup)
         @write_buffer_bytes += bytes.bytesize
         flush_write_buffer if @write_buffer_bytes >= WRITE_BUFFER_FLUSH_BYTES
         bytes.bytesize
@@ -282,7 +285,11 @@ module Dalli
       end
 
       def fork_detected?
-        @pid && @pid != PIDCache.pid
+        # Process.pid rather than PIDCache: PIDCache is refreshed by a
+        # Process._fork hook, and another library's fork hook (such as
+        # connection_pool closing its connections) can run in the child before
+        # it, which would close the parent's TLS session as if it were ours.
+        @pid && @pid != Process.pid
       end
 
       def log_down_detected
@@ -306,13 +313,32 @@ module Dalli
 
       private
 
+      # One write per flush. A TLS socket sends each argument of
+      # write(*parts) as its own record and system call, so the parts are
+      # joined first.
       def flush_write_buffer
         return if @write_buffer.empty?
 
-        @sock.write(*@write_buffer)
+        @sock.write(@write_buffer.size == 1 ? @write_buffer.first : joined_write_buffer)
         discard_write_buffer
       end
 
+      # Joined as bytes: requests can mix UTF-8 and binary strings, which
+      # String#<< refuses to combine when both hold non-ASCII bytes
+      if String.method_defined?(:append_as_bytes)
+        def joined_write_buffer
+          out = String.new(capacity: @write_buffer_bytes, encoding: Encoding::BINARY)
+          @write_buffer.each { |part| out.append_as_bytes(part) }
+          out
+        end
+      else
+        def joined_write_buffer
+          out = String.new(capacity: @write_buffer_bytes, encoding: Encoding::BINARY)
+          @write_buffer.each { |part| out << part.b }
+          out
+        end
+      end
+
       def discard_write_buffer
         @write_buffer.clear
         @write_buffer_bytes = 0

data/lib/dalli/protocol/meta/response_processor.rb

@@ -28,22 +28,25 @@ module Dalli
           @value_marshaller = value_marshaller
         end
 
-        def meta_get_with_value(cache_nils: false)
+        # raw: true (a request made with raw: true) returns the value as stored,
+        # ignoring any flags in the reply. The request didn't ask for flags, so
+        # a reply carrying them anyway must not get the value deserialized.
+        def meta_get_with_value(cache_nils: false, raw: false)
           tokens = error_on_unexpected!([VA, EN, HD])
           return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
           return true unless tokens.first == VA
 
-          @value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
+          retrieve(read_data(tokens[1].to_i), raw ? 0 : bitflags_from_tokens(tokens), raw)
         end
 
-        def meta_get_with_value_and_cas
+        def meta_get_with_value_and_cas(raw: false)
           tokens = error_on_unexpected!([VA, EN, HD])
           return [nil, 0] if tokens.first == EN
 
           cas = cas_from_tokens(tokens)
           return [nil, cas] unless tokens.first == VA
 
-          [@value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens)), cas]
+          [retrieve(read_data(tokens[1].to_i), raw ? 0 : bitflags_from_tokens(tokens), raw), cas]
         end
 
         def meta_get_without_value
@@ -62,12 +65,13 @@ module Dalli
         #
         # Used by meta_get for comprehensive metadata retrieval.
         # Supports thundering herd protection (N/R flags) and metadata flags (h/l/u).
-        def meta_get_with_metadata(cache_nils: false, return_hit_status: false, return_last_access: false)
+        def meta_get_with_metadata(cache_nils: false, return_hit_status: false, return_last_access: false,
+                                   raw: false)
           tokens = error_on_unexpected!([VA, EN, HD])
           result = build_metadata_result(tokens)
           result[:hit_before] = hit_status_from_tokens(tokens) if return_hit_status
           result[:last_access] = last_access_from_tokens(tokens) if return_last_access
-          result[:value] = parse_value_from_tokens(tokens, cache_nils)
+          result[:value] = parse_value_from_tokens(tokens, cache_nils, raw)
           result
         end
 
@@ -79,11 +83,16 @@ module Dalli
           }
         end
 
-        def parse_value_from_tokens(tokens, cache_nils)
+        def parse_value_from_tokens(tokens, cache_nils, raw = false) # rubocop:disable Style/OptionalBooleanParameter
           return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
           return unless tokens.first == VA
 
-          @value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
+          retrieve(read_data(tokens[1].to_i), raw ? 0 : bitflags_from_tokens(tokens), raw)
+        end
+
+        # Values from a raw request are returned as stored
+        def retrieve(value, bitflags, raw)
+          raw ? value : @value_marshaller.retrieve(value, bitflags)
         end
 
         def meta_set_with_cas
@@ -148,6 +157,10 @@ module Dalli
         end
 
         def full_response_from_buffer(tokens, body, resp_size)
+          # A hit with no key can't be matched to a request: skip it (false
+          # status) rather than yield it, or take it for the end of the pipeline
+          return [resp_size, false, nil, nil, nil] unless tokens.any? { |t| t.start_with?('k') }
+
           value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
           [resp_size, tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value]
         end
@@ -197,6 +210,11 @@ module Dalli
         # buffer wait for (and accumulate) that many bytes.
         MAX_VALUE_BYTES = 1024 * 1024 * 1024
 
+        # The size given right after "VA", or 0 when it isn't a number
+        def positional_size(tokens)
+          tokens.first == VA && tokens[1]&.match?(/\A\d+\z/) ? tokens[1].to_i : 0
+        end
+
         def check_value_size!(size)
           return if size.between?(0, MAX_VALUE_BYTES)
 
@@ -248,7 +266,9 @@ module Dalli
         end
 
         def body_len_from_tokens(tokens)
-          size = value_from_tokens(tokens, 's')&.to_i
+          # memcached also gives the size positionally ("VA <size> ..."); use it
+          # when there's no s flag rather than reading the value as replies
+          size = tokens.any? { |t| t.start_with?('s') } ? value_from_tokens(tokens, 's')&.to_i : positional_size(tokens)
           check_value_size!(size) if size
           size
         end
@@ -270,6 +290,9 @@ module Dalli
         end
 
         def read_data(data_size)
+          # Checked before the terminator is added: -1 or -2 would otherwise
+          # read 1 or 0 bytes and leave the connection out of step
+          check_value_size!(data_size)
           @io_source.read(data_size + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
         end
       end

data/lib/dalli/protocol/meta.rb

@@ -30,7 +30,7 @@ module Dalli
         req = RequestFormatter.meta_get(key: encoded_key, base64: base64, skip_flags: skip_flags)
         write(req)
         @connection_manager.flush
-        response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
+        response_processor.meta_get_with_value(cache_nils: cache_nils?(options), raw: raw_request?(options))
       end
 
       def quiet_get_request(key)
@@ -47,7 +47,7 @@ module Dalli
         req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, base64: base64, skip_flags: skip_flags)
         write(req)
         @connection_manager.flush
-        response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
+        response_processor.meta_get_with_value(cache_nils: cache_nils?(options), raw: raw_request?(options))
       end
 
       def touch(key, ttl)
@@ -61,12 +61,13 @@ module Dalli
 
       # TODO: This is confusing, as there's a cas command in memcached
       # and this isn't it.  Maybe rename?  Maybe eliminate?
-      def cas(key)
+      def cas(key, options = nil)
         encoded_key, base64 = KeyRegularizer.encode(key)
-        req = RequestFormatter.meta_get(key: encoded_key, value: true, return_cas: true, base64: base64)
+        req = RequestFormatter.meta_get(key: encoded_key, value: true, return_cas: true, base64: base64,
+                                        skip_flags: raw_request?(options))
         write(req)
         @connection_manager.flush
-        response_processor.meta_get_with_value_and_cas
+        response_processor.meta_get_with_value_and_cas(raw: raw_request?(options))
       end
 
       # Comprehensive meta get with support for all metadata flags.
@@ -105,7 +106,7 @@ module Dalli
         @connection_manager.flush
         response_processor.meta_get_with_metadata(
           cache_nils: cache_nils?(options), return_hit_status: options[:return_hit_status],
-          return_last_access: options[:return_last_access]
+          return_last_access: options[:return_last_access], raw: raw_request?(options)
         )
       end
 

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '4.3.6'
+  VERSION = '4.3.7'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 4.3.6
+  version: 4.3.7
 platform: ruby
 authors:
 - Peter M. Goldstein