dalli 5.1.4 → 5.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70f17667b9c623f9233be342d920de385bab797b44a4bc9f1e85c233f766404f
-  data.tar.gz: 527b9ff3dd4699ad5d307107668b31d3a739675825ad4d0b2cfb35ac09581a81
+  metadata.gz: fd6ca01dc58cadc97706b152f76a3bff1bb94a4a4727743cddac5870b6120332
+  data.tar.gz: ef6123de8d8267c5ded812f05fda7d4a84cb1b56c116afd3fe8bd00365881175
 SHA512:
-  metadata.gz: 9e54f454a1bca3b79f229a219b458533d2778c1521043c1ee7c2d203ae0d25e1449fc375b59754aaadac79de6ea484170ba52c10008da6abd40451821300a60c
-  data.tar.gz: 1dea61992e48d8786dbb3ef929e30c53d3544bafed895334cf4316ad0710791c4ebb0887fb4fa63b79358c0029be11b3495a2875a24956a296246fe9d5e4eaa0
+  metadata.gz: 16082586633b6f84d1486e82a603b4d7fdc1afb586bf67c0a9f6ede7d9b426a87ec0b2b59eef019c471568ad5f0a8bd081ddb164c08d608387c9093d3dba25d7
+  data.tar.gz: 15788269cf8910b1bee979b7859f57279f9723f57407c6c6b4dac79a60fe454d00e896e7514dc93de7a90366ce9b071d4308fabbad7e41f3f1b71593e376a6b2

data/CHANGELOG.md

@@ -4,6 +4,17 @@ Dalli Changelog
 Unreleased
 ==========
 
+5.1.5
+==========
+
+Security:
+
+- Don't rely on memcached's auto-create for keys that need base64 encoding (GHSA-r47x-6p6r-mvcp)
+  - memcached stores an item made by auto-create (`ma` with `N`/`J`, or `mg` with `N`) for a base64-encoded key without its binary-key flag, and later replies name the item by its raw key bytes. A key containing spaces or CR/LF could then make a multi-get read the replies after it as different ones
+  - `incr` and `decr` with an initial value now create such counters with an add instead. Return values are unchanged
+  - The new `verify_reply_keys` option makes `get_multi`, `get_multi_cas` and `get_multi_with_metadata` ignore replies for keys they didn't request, which guards against items created by older Dalli versions or other clients until memcached is upgraded. It's off by default, since it costs a multi-get about 15%
+  - Affects the meta protocol since 3.2.0; fixed in 5.2.3, 5.1.5, 5.0.11, 4.3.8 and 3.2.14. memcached 1.6.45 and earlier still name these items by their raw key bytes
+
 5.1.4
 ==========
 

data/README.md

@@ -63,6 +63,10 @@ Per-request `raw: true` returns the stored bytes without deserializing or decomp
 
 See the [5.0-Upgrade.md](5.0-Upgrade.md) guide for upgrade information.
 
+### Counters and verify_reply_keys
+
+memcached names an item made by auto-create for a base64-encoded key (one with whitespace, control or non-ASCII characters) by its raw key bytes in later replies. Since 5.1.5, Dalli creates counters for such keys with an add rather than auto-create. Items created before then, or by other clients, can still be named that way until memcached releases a fix (1.6.45 and earlier behave this way). If keys passed to `incr` or `decr` with an initial value can contain untrusted input, set `verify_reply_keys: true` until then: multi-gets then ignore replies for keys they didn't request, at a cost of about 15% per multi-get.
+
 ## OpenTelemetry Tracing
 
 Dalli automatically instruments operations with [OpenTelemetry](https://opentelemetry.io/) when the SDK is present. No configuration is required - just add the OpenTelemetry gems to your application:

data/lib/dalli/client.rb

@@ -50,6 +50,11 @@ module Dalli
     #                        +:include+ logs the full operation and key(s), +:obfuscate+ replaces keys with "?",
     #                        +nil+ (default) omits the attribute entirely.
     # - :otel_peer_service - when set, adds a +peer.service+ span attribute with this value for logical service naming.
+    # - :verify_reply_keys - defaults to false. When true, get_multi, get_multi_cas and get_multi_with_metadata only
+    #                        accept replies for the keys they requested; after a reply naming another key, the rest of
+    #                        that server's replies are treated as misses. memcached can name an item by its raw key
+    #                        bytes when the item was created by auto-create from a base64-encoded key, which Dalli no
+    #                        longer does. Costs a multi-get about 15%.
     #
     def initialize(servers = nil, options = {})
       @normalized_servers = ::Dalli::ServersArgNormalizer.normalize_servers(servers)
@@ -745,6 +750,11 @@ module Dalli
       Dalli.logger.debug { e.inspect }
       Dalli.logger.debug { 'retrying single-server get_multi because of network error' }
       retry
+    rescue Dalli::UnexpectedReplyKeyError => e
+      # The connection is closed and the keys are treated as missing, as the
+      # pipelined get_multi does for a server whose replies are out of step
+      Dalli.logger.warn { e.message }
+      {}
     end
 
     def single_server_get_multi_with_metadata(keys, req_options = nil)
@@ -756,6 +766,9 @@ module Dalli
       result
     rescue Dalli::NetworkError
       {}
+    rescue Dalli::UnexpectedReplyKeyError => e
+      Dalli.logger.warn { e.message }
+      {}
     end
 
     def single_server_set_multi(hash, ttl, req_options)

data/lib/dalli/protocol/base.rb

@@ -41,6 +41,7 @@ module Dalli
         warn_uri_credentials(user_creds)
         @options = client_options.merge(user_creds)
         @raw_mode = client_options[:raw]
+        @verify_reply_keys = client_options[:verify_reply_keys] ? true : false
         @value_marshaller = @raw_mode ? StringMarshaller.new(@options) : ValueMarshaller.new(@options)
         @connection_manager = ConnectionManager.new(hostname, port, socket_type, @options)
       end
@@ -147,6 +148,8 @@ module Dalli
       # avoiding intermediate Hash allocation. Returns nil.
       # Without a block, returns a Hash of { key => [value, cas] }.
       def pipeline_next_responses(&block)
+        return abandon_pipeline if @pipeline_out_of_step
+
         reconnect_on_pipeline_complete!
         values = nil
 
@@ -168,6 +171,13 @@ module Dalli
           # to the noop at the end of the pipeline
           finish_pipeline && break if status && key.nil?
 
+          # A key that wasn't requested means the replies are out of step
+          # (see expect_reply_keys): keep what was read so far, drop the rest
+          if unexpected_reply_key?(key)
+            abandon_pipeline
+            break
+          end
+
           # If the status is ok and the key is not nil, then this is a
           # getkq response with a value that we want to set in the response hash
           unless key.nil?
@@ -332,6 +342,7 @@ module Dalli
         # the expected state before pipeline_response_setup is called.
         response_buffer.clear
         response_buffer.raw = raw_request?(options)
+        expect_reply_keys(keys)
 
         # The terminating noop is sent by pipeline_response_setup
         write(quiet_get_requests(keys, options, return_cas: return_cas))
@@ -344,6 +355,7 @@ module Dalli
         # Initialize the response buffer for draining during send phase
         response_buffer.ensure_ready
         response_buffer.raw = raw_request?(options)
+        expect_reply_keys(keys)
 
         keys.each_slice(chunk_size) do |chunk|
           # Build and write this chunk of requests
@@ -376,6 +388,12 @@ module Dalli
           status, cas, key, value = response_buffer.process_single_getk_response
           break if status.nil? # No complete response available
 
+          # Handled by the next pipeline_next_responses, once sending is done
+          if unexpected_reply_key?(key)
+            @pipeline_out_of_step = true
+            break
+          end
+
           results[key] = [value, cas] unless key.nil?
         end
       rescue SystemCallError, Dalli::NetworkError
@@ -387,6 +405,42 @@ module Dalli
         @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
       end
 
+      # A multi-get only accepts replies for the keys it asked for. memcached
+      # names an item created by auto-create (ma or mg with N) for a
+      # base64-encoded key by its raw bytes, without the b flag; such a key
+      # can contain spaces or CR/LF, which split the reply header and make the
+      # replies after it parse as different ones. A key that wasn't requested
+      # is how that shows up.
+      # Only with the verify_reply_keys option, since building the set costs a
+      # multi-get about 15%. Dalli itself no longer creates such items (see
+      # Meta#decr_incr); the check guards against ones that older versions or
+      # other clients created, on memcached versions with the behavior.
+      def expect_reply_keys(keys)
+        @pipeline_out_of_step = false
+        @expected_reply_keys = (keys.to_set { |k| reply_key_form(k) } if @verify_reply_keys)
+      end
+
+      def unexpected_reply_key?(key)
+        !key.nil? && @expected_reply_keys && !@expected_reply_keys.include?(reply_key_form(key))
+      end
+
+      # Keys compared as bytes; ASCII keys compare equal in any encoding as is
+      def reply_key_form(key)
+        key.ascii_only? ? key : key.b
+      end
+
+      # Keeps what was read, drops the rest of this server's replies, and
+      # closes the connection so nothing unread is left on it. Not a socket
+      # failure, so it doesn't count toward socket_max_failures; the keys not
+      # read are simply missing from the result.
+      def abandon_pipeline
+        Dalli.logger.warn { "#{name} replied to a get_multi with a key that wasn't requested; closing the connection" }
+        @pipeline_out_of_step = false
+        response_buffer.clear
+        @connection_manager.close
+        {}
+      end
+
       # Called after the noop response is received at the end of a set
       # of pipelined gets
       def finish_pipeline

data/lib/dalli/protocol/meta.rb

@@ -224,12 +224,45 @@ module Dalli
       # rubocop:disable-next Metrics/ParameterLists
       def decr_incr(incr, key, delta, ttl, initial, options = nil)
         ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
+        # memcached stores an item created by ma's auto-create (N/J) for a
+        # base64-encoded key without its binary-key flag, and later replies
+        # name it with the raw key bytes. Create such counters with an add.
+        return decr_incr_with_add(incr, key, delta, ttl, initial, options) if initial && KeyRegularizer.required?(key)
+
         write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: initial, incr: incr, ttl: ttl,
                                                quiet: quiet?, **routing_token_kwargs(options)))
         @connection_manager.flush unless quiet?
         response_processor.decr_incr unless quiet?
       end
 
+      # Like decr_incr with an initial value, without memcached's auto-create:
+      # on a miss the counter is added with the initial value (which is
+      # returned, as auto-create would), and the add losing a race to another
+      # client means the counter now exists, so the incr is tried again. In a
+      # quiet block both requests are sent and whichever applies takes effect.
+      # rubocop:disable-next Metrics/ParameterLists
+      def decr_incr_with_add(incr, key, delta, ttl, initial, options)
+        add_options = (options || {}).merge(raw: true) # counters are stored raw
+        return quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options) if quiet?
+
+        2.times do
+          value = decr_incr(incr, key, delta, nil, nil, options)
+          return value unless value.nil?
+
+          write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: false)
+          return initial if response_processor.meta_set_with_cas
+        end
+        decr_incr(incr, key, delta, nil, nil, options)
+      end
+
+      # rubocop:disable-next Metrics/ParameterLists
+      def quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options)
+        write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: nil, incr: incr, ttl: nil,
+                                               quiet: true, **routing_token_kwargs(add_options)))
+        # Sent with the rest of the quiet block, by its closing noop
+        write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: true)
+      end
+
       # Other Commands
       def flush(delay = 0)
         write(RequestFormatter.flush(delay: delay))
@@ -271,7 +304,7 @@ module Dalli
         buffer = RequestFormatter.multi_meta_get(keys, skip_flags: is_raw, **routing_token_kwargs(options))
         flushed_write(buffer)
         buffer.clear
-        read_multi_get_responses(is_raw)
+        read_multi_get_responses(is_raw, keys)
       end
 
       # Stale-aware bulk get.  Returns { key => { value:, cas:, stale:, miss: } }
@@ -288,12 +321,13 @@ module Dalli
                                                        **routing_token_kwargs(options))
         flushed_write(buffer)
         buffer.clear
-        read_multi_metadata_responses(is_raw)
+        read_multi_metadata_responses(is_raw, keys)
       end
 
       # Unlike read_multi_get_responses this locates tokens by flag rather than
       # by position, because the c flag shifts the key's index.
-      def read_multi_metadata_responses(is_raw)
+      def read_multi_metadata_responses(is_raw, keys)
+        expect_reply_keys(keys)
         hash = {}
         while (line = @connection_manager.read_line)
           break if line.start_with?('MN')
@@ -309,12 +343,15 @@ module Dalli
           key = response_processor.key_from_tokens(tokens)
           next if key.nil?
 
+          raise_unexpected_reply_key(key) if unexpected_reply_key?(key)
+
           hash[key] = { value: @value_marshaller.retrieve(value, bitflags), cas: cas, stale: stale, miss: false }
         end
         hash
       end
 
-      def read_multi_get_responses(is_raw)
+      def read_multi_get_responses(is_raw, keys)
+        expect_reply_keys(keys)
         hash = {}
         key_index = is_raw ? 2 : 3
         while (line = @connection_manager.read_line)
@@ -322,11 +359,19 @@ module Dalli
           next unless line.start_with?('VA ')
 
           key, value = parse_multi_get_value(line, key_index, is_raw)
+          raise_unexpected_reply_key(key) if unexpected_reply_key?(key)
           hash[key] = value if key
         end
         hash
       end
 
+      # The replies after a key that wasn't requested can't be trusted (see
+      # Base#expect_reply_keys). Raising ends the request, which closes the
+      # connection; the caller treats this server's keys as missing.
+      def raise_unexpected_reply_key(key)
+        raise Dalli::UnexpectedReplyKeyError, "#{name} replied with a key that wasn't requested: #{key.inspect}"
+      end
+
       def parse_multi_get_value(line, key_index, is_raw)
         tokens = line.chomp!(TERMINATOR).split
         size = tokens[1].to_i

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '5.1.4'
+  VERSION = '5.1.5'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.6.27'
 end

data/lib/dalli.rb

@@ -31,6 +31,10 @@ module Dalli
   # socket/server communication error that can be retried
   class RetryableNetworkError < NetworkError; end
 
+  # a multi-get reply named a key that wasn't requested, so the rest of that
+  # server's replies can't be trusted
+  class UnexpectedReplyKeyError < DalliError; end
+
   # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
   class NilObject; end # rubocop:disable Lint/EmptyClass
   NOT_FOUND = NilObject.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 5.1.4
+  version: 5.1.5
 platform: ruby
 authors:
 - Peter M. Goldstein
@@ -88,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.21
+rubygems_version: 4.0.23
 specification_version: 4
 summary: High performance memcached client for Ruby
 test_files: []