dalli 5.0.10 → 5.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64bc18f470e5b97d87b6531a968a9252c4ff46ba715d927c74b66e52f7549683
-  data.tar.gz: 70fc9db7148b82932b9f3323ac74a2c59dd7ed8b72bd9543dc160c5026d33880
+  metadata.gz: 43164df9e3e36d747346b72c44ebe1a7a3be0dee501c83b2ef813a39d14d9bb9
+  data.tar.gz: e8e1ecbf3d019891da3935c2d0c805f09bfc8a9746df14d9a6954f8cdecfdc10
 SHA512:
-  metadata.gz: a2c3def53ce7c5c4bf4d929bc45c5d1ff7f000db6b59abea3ac9ef3eb81e7f706ab8d0b01021b7002ff2b2153014b17c018c5060935e022f867b4981318afc06
-  data.tar.gz: f63a580cc08950f6da91213f0c4b9ecd2cb75961fe4467e06c11ecb434d2b13ff5950fceb31d43f8b8c42d8997fec3fd1bd40eebc1aa59dda41b2c1092a5e253
+  metadata.gz: e5b73ad3b6a075030a75f7f7903a097611cfd862f65a80a549d11f1c34c900205ac4ed4783392be4ee139c598b6de346ddaf782460c0ec11e8fc31847b4cb4aa
+  data.tar.gz: 5978a35d174e82c4cdc1dafdd439395f64035517f5a4592ee0ad1724e6b9b4241551c5b5990e6a6823d433e68604697bb43940e8bf3b82b15952325264e8b446

data/CHANGELOG.md

@@ -4,6 +4,17 @@ Dalli Changelog
 Unreleased
 ==========
 
+5.0.11
+==========
+
+Security:
+
+- Don't rely on memcached's auto-create for keys that need base64 encoding (GHSA-r47x-6p6r-mvcp)
+  - memcached stores an item made by auto-create (`ma` with `N`/`J`, or `mg` with `N`) for a base64-encoded key without its binary-key flag, and later replies name the item by its raw key bytes. A key containing spaces or CR/LF could then make a multi-get read the replies after it as different ones
+  - `incr` and `decr` with an initial value now create such counters with an add instead. Return values are unchanged
+  - The new `verify_reply_keys` option makes `get_multi` and `get_multi_cas` ignore replies for keys they didn't request, which guards against items created by older Dalli versions or other clients until memcached is upgraded. It's off by default, since it costs a multi-get about 15%
+  - Affects the meta protocol since 3.2.0; fixed in 5.2.3, 5.1.5, 5.0.11, 4.3.8 and 3.2.14. memcached 1.6.45 and earlier still name these items by their raw key bytes
+
 5.0.10
 ==========
 

data/README.md

@@ -58,6 +58,10 @@ Per-request `raw: true` returns the stored bytes without deserializing or decomp
 
 See the [5.0-Upgrade.md](5.0-Upgrade.md) guide for upgrade information.
 
+### Counters and verify_reply_keys
+
+memcached names an item made by auto-create for a base64-encoded key (one with whitespace, control or non-ASCII characters) by its raw key bytes in later replies. Since 5.0.11, Dalli creates counters for such keys with an add rather than auto-create. Items created before then, or by other clients, can still be named that way until memcached releases a fix (1.6.45 and earlier behave this way). If keys passed to `incr` or `decr` with an initial value can contain untrusted input, set `verify_reply_keys: true` until then: multi-gets then ignore replies for keys they didn't request, at a cost of about 15% per multi-get.
+
 ## OpenTelemetry Tracing
 
 Dalli automatically instruments operations with [OpenTelemetry](https://opentelemetry.io/) when the SDK is present. No configuration is required - just add the OpenTelemetry gems to your application:

data/lib/dalli/client.rb

@@ -50,6 +50,11 @@ module Dalli
     #                        +:include+ logs the full operation and key(s), +:obfuscate+ replaces keys with "?",
     #                        +nil+ (default) omits the attribute entirely.
     # - :otel_peer_service - when set, adds a +peer.service+ span attribute with this value for logical service naming.
+    # - :verify_reply_keys - defaults to false. When true, get_multi, get_multi_cas and get_multi_with_metadata only
+    #                        accept replies for the keys they requested; after a reply naming another key, the rest of
+    #                        that server's replies are treated as misses. memcached can name an item by its raw key
+    #                        bytes when the item was created by auto-create from a base64-encoded key, which Dalli no
+    #                        longer does. Costs a multi-get about 15%.
     #
     def initialize(servers = nil, options = {})
       @normalized_servers = ::Dalli::ServersArgNormalizer.normalize_servers(servers)
@@ -572,6 +577,9 @@ module Dalli
       result
     rescue Dalli::NetworkError
       {}
+    rescue Dalli::UnexpectedReplyKeyError => e
+      Dalli.logger.warn { e.message }
+      {}
     end
 
     def single_server_set_multi(hash, ttl, req_options)

data/lib/dalli/protocol/base.rb

@@ -25,6 +25,7 @@ module Dalli
         warn_uri_credentials(user_creds)
         @options = client_options.merge(user_creds)
         @raw_mode = client_options[:raw]
+        @verify_reply_keys = client_options[:verify_reply_keys] ? true : false
         @value_marshaller = @raw_mode ? StringMarshaller.new(@options) : ValueMarshaller.new(@options)
         @connection_manager = ConnectionManager.new(hostname, port, socket_type, @options)
       end
@@ -118,6 +119,8 @@ module Dalli
       # avoiding intermediate Hash allocation. Returns nil.
       # Without a block, returns a Hash of { key => [value, cas] }.
       def pipeline_next_responses(&block)
+        return abandon_pipeline if @pipeline_out_of_step
+
         reconnect_on_pipeline_complete!
         values = nil
 
@@ -139,6 +142,13 @@ module Dalli
           # to the noop at the end of the pipeline
           finish_pipeline && break if status && key.nil?
 
+          # A key that wasn't requested means the replies are out of step
+          # (see expect_reply_keys): keep what was read so far, drop the rest
+          if unexpected_reply_key?(key)
+            abandon_pipeline
+            break
+          end
+
           # If the status is ok and the key is not nil, then this is a
           # getkq response with a value that we want to set in the response hash
           unless key.nil?
@@ -265,6 +275,7 @@ module Dalli
         # Use clear (not reset) to keep pipeline_complete? = true, which is
         # the expected state before pipeline_response_setup is called.
         response_buffer.clear
+        expect_reply_keys(keys)
 
         req = +''
         keys.each do |key|
@@ -280,6 +291,7 @@ module Dalli
       def pipelined_get_interleaved(keys, chunk_size, results)
         # Initialize the response buffer for draining during send phase
         response_buffer.ensure_ready
+        expect_reply_keys(keys)
 
         keys.each_slice(chunk_size) do |chunk|
           # Build and write this chunk of requests
@@ -312,6 +324,12 @@ module Dalli
           status, cas, key, value = response_buffer.process_single_getk_response
           break if status.nil? # No complete response available
 
+          # Handled by the next pipeline_next_responses, once sending is done
+          if unexpected_reply_key?(key)
+            @pipeline_out_of_step = true
+            break
+          end
+
           results[key] = [value, cas] unless key.nil?
         end
       rescue SystemCallError, Dalli::NetworkError
@@ -323,6 +341,42 @@ module Dalli
         @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
       end
 
+      # A multi-get only accepts replies for the keys it asked for. memcached
+      # names an item created by auto-create (ma or mg with N) for a
+      # base64-encoded key by its raw bytes, without the b flag; such a key
+      # can contain spaces or CR/LF, which split the reply header and make the
+      # replies after it parse as different ones. A key that wasn't requested
+      # is how that shows up.
+      # Only with the verify_reply_keys option, since building the set costs a
+      # multi-get about 15%. Dalli itself no longer creates such items (see
+      # Meta#decr_incr); the check guards against ones that older versions or
+      # other clients created, on memcached versions with the behavior.
+      def expect_reply_keys(keys)
+        @pipeline_out_of_step = false
+        @expected_reply_keys = (keys.to_set { |k| reply_key_form(k) } if @verify_reply_keys)
+      end
+
+      def unexpected_reply_key?(key)
+        !key.nil? && @expected_reply_keys && !@expected_reply_keys.include?(reply_key_form(key))
+      end
+
+      # Keys compared as bytes; ASCII keys compare equal in any encoding as is
+      def reply_key_form(key)
+        key.ascii_only? ? key : key.b
+      end
+
+      # Keeps what was read, drops the rest of this server's replies, and
+      # closes the connection so nothing unread is left on it. Not a socket
+      # failure, so it doesn't count toward socket_max_failures; the keys not
+      # read are simply missing from the result.
+      def abandon_pipeline
+        Dalli.logger.warn { "#{name} replied to a get_multi with a key that wasn't requested; closing the connection" }
+        @pipeline_out_of_step = false
+        response_buffer.clear
+        @connection_manager.close
+        {}
+      end
+
       # Called after the noop response is received at the end of a set
       # of pipelined gets
       def finish_pipeline

data/lib/dalli/protocol/meta.rb

@@ -193,12 +193,44 @@ module Dalli
 
       def decr_incr(incr, key, delta, ttl, initial)
         ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
+        # memcached stores an item created by ma's auto-create (N/J) for a
+        # base64-encoded key without its binary-key flag, and later replies
+        # name it with the raw key bytes. Create such counters with an add.
+        return decr_incr_with_add(incr, key, delta, ttl, initial) if initial && KeyRegularizer.required?(key)
+
         write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: initial, incr: incr, ttl: ttl,
                                                quiet: quiet?))
         @connection_manager.flush unless quiet?
         response_processor.decr_incr unless quiet?
       end
 
+      # Like decr_incr with an initial value, without memcached's auto-create:
+      # on a miss the counter is added with the initial value (which is
+      # returned, as auto-create would), and the add losing a race to another
+      # client means the counter now exists, so the incr is tried again. In a
+      # quiet block both requests are sent and whichever applies takes effect.
+      def decr_incr_with_add(incr, key, delta, ttl, initial)
+        add_options = { raw: true } # counters are stored raw
+        return quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options) if quiet?
+
+        2.times do
+          value = decr_incr(incr, key, delta, nil, nil)
+          return value unless value.nil?
+
+          write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: false)
+          return initial if response_processor.meta_set_with_cas
+        end
+        decr_incr(incr, key, delta, nil, nil)
+      end
+
+      # rubocop:disable-next Metrics/ParameterLists
+      def quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options)
+        write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: nil, incr: incr, ttl: nil,
+                                               quiet: true))
+        # Sent with the rest of the quiet block, by its closing noop
+        write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: true)
+      end
+
       # Other Commands
       def flush(delay = 0)
         write(RequestFormatter.flush(delay: delay))
@@ -240,10 +272,11 @@ module Dalli
         buffer = RequestFormatter.multi_meta_get(keys, skip_flags: is_raw)
         flushed_write(buffer)
         buffer.clear
-        read_multi_get_responses(is_raw)
+        read_multi_get_responses(is_raw, keys)
       end
 
-      def read_multi_get_responses(is_raw)
+      def read_multi_get_responses(is_raw, keys)
+        expect_reply_keys(keys)
         hash = {}
         key_index = is_raw ? 2 : 3
         while (line = @connection_manager.read_line)
@@ -251,11 +284,19 @@ module Dalli
           next unless line.start_with?('VA ')
 
           key, value = parse_multi_get_value(line, key_index, is_raw)
+          raise_unexpected_reply_key(key) if unexpected_reply_key?(key)
           hash[key] = value if key
         end
         hash
       end
 
+      # The replies after a key that wasn't requested can't be trusted (see
+      # Base#expect_reply_keys). Raising ends the request, which closes the
+      # connection; the caller treats this server's keys as missing.
+      def raise_unexpected_reply_key(key)
+        raise Dalli::UnexpectedReplyKeyError, "#{name} replied with a key that wasn't requested: #{key.inspect}"
+      end
+
       def parse_multi_get_value(line, key_index, is_raw)
         tokens = line.chomp!(TERMINATOR).split
         size = tokens[1].to_i

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '5.0.10'
+  VERSION = '5.0.11'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.6'
 end

data/lib/dalli.rb

@@ -31,6 +31,10 @@ module Dalli
   # socket/server communication error that can be retried
   class RetryableNetworkError < NetworkError; end
 
+  # a multi-get reply named a key that wasn't requested, so the rest of that
+  # server's replies can't be trusted
+  class UnexpectedReplyKeyError < DalliError; end
+
   # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
   class NilObject; end # rubocop:disable Lint/EmptyClass
   NOT_FOUND = NilObject.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 5.0.10
+  version: 5.0.11
 platform: ruby
 authors:
 - Peter M. Goldstein
@@ -88,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.21
+rubygems_version: 4.0.23
 specification_version: 4
 summary: High performance memcached client for Ruby
 test_files: []