dalli 4.3.7 → 4.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b1a68fbdf9ea89024d6e9bc5803f94d3342c2f7b8bd7cfea3a97f7a873f7b76
-  data.tar.gz: 93d1c2fcb71fa739e6a00808cddb11567ad9fed3caa4d0751c924576bc04bfe0
+  metadata.gz: 7cddc5e9ac0bd88077676d94695d72804971c1b5a65522e3f8bff05b9b35f75a
+  data.tar.gz: 7c506e73dd7b0c71864aaa99bb35537642dd966ad692101556be733050891c1b
 SHA512:
-  metadata.gz: bc42cde8b6f61ba60768bf43a9ebf8b1088dc8b2efbb4e32b5ed1abedd3c3be81c66aa02dc1a2558fd319debcc8074e61471996ad8825afed32d4fd741f8a312
-  data.tar.gz: 3a8320685a39edef196feaa387ef954caea41650b540c7f37d32d4821b5804f41631e5346e14b21912de0963f147c7d55e8c1ad08e24d2f7d25073975ba64d72
+  metadata.gz: db49da025cb40d2902dbd394d9c2ee1f3839e41b912f1e3d8a2f3f0f096e409e4ea564a8f9783677b00b5307aef4c64951818b385af66d8c958103a36e89f076
+  data.tar.gz: a8d55c26b215141a5af0f59782e1e8a3443f07b73a27175bc39cde8bda798c176e88c658b8cebad14c52198d974fd30ae6e859f321102cdf6d738b7157fc5bbe

data/CHANGELOG.md

@@ -4,6 +4,17 @@ Dalli Changelog
 Unreleased
 ==========
 
+4.3.8
+==========
+
+Security:
+
+- Don't rely on memcached's auto-create for keys that need base64 encoding (GHSA-r47x-6p6r-mvcp)
+  - memcached stores an item made by auto-create (`ma` with `N`/`J`, or `mg` with `N`) for a base64-encoded key without its binary-key flag, and later replies name the item by its raw key bytes. A key containing spaces or CR/LF could then make a multi-get read the replies after it as different ones
+  - `incr` and `decr` with an initial value now create such counters with an add instead. Return values are unchanged
+  - The new `verify_reply_keys` option makes `get_multi` and `get_multi_cas` ignore replies for keys they didn't request, which guards against items created by older Dalli versions or other clients until memcached is upgraded. It's off by default, since it costs a multi-get about 15%
+  - Affects `protocol: :meta` since 3.2.0; fixed in 5.2.3, 5.1.5, 5.0.11, 4.3.8 and 3.2.14. memcached 1.6.45 and earlier still name these items by their raw key bytes
+
 4.3.7
 ==========
 

data/README.md

@@ -82,6 +82,10 @@ Per-request `raw: true` returns the stored bytes without deserializing or decomp
 
 See the [4.0-Upgrade.md](4.0-Upgrade.md) guide for more information.
 
+### Counters and verify_reply_keys
+
+memcached names an item made by auto-create for a base64-encoded key (one with whitespace, control or non-ASCII characters) by its raw key bytes in later replies (with `protocol: :meta`). Since 4.3.8, Dalli creates counters for such keys with an add rather than auto-create. Items created before then, or by other clients, can still be named that way until memcached releases a fix (1.6.45 and earlier behave this way). If keys passed to `incr` or `decr` with an initial value can contain untrusted input, set `verify_reply_keys: true` until then: multi-gets then ignore replies for keys they didn't request, at a cost of about 15% per multi-get.
+
 ## OpenTelemetry Tracing
 
 Dalli automatically instruments operations with [OpenTelemetry](https://opentelemetry.io/) when the SDK is present. No configuration is required - just add the OpenTelemetry gems to your application:

data/lib/dalli/client.rb

@@ -53,6 +53,11 @@ module Dalli
     #                        +:include+ logs the full operation and key(s), +:obfuscate+ replaces keys with "?",
     #                        +nil+ (default) omits the attribute entirely.
     # - :otel_peer_service - when set, adds a +peer.service+ span attribute with this value for logical service naming.
+    # - :verify_reply_keys - defaults to false. When true, get_multi and get_multi_cas only accept replies for the keys
+    #                        they requested; after a reply naming another key, the rest of that server's replies are
+    #                        treated as misses. With the meta protocol, memcached can name an item by its raw key bytes
+    #                        when the item was created by auto-create from a base64-encoded key, which Dalli no longer
+    #                        does. Costs a multi-get about 15%.
     #
     def initialize(servers = nil, options = {})
       @normalized_servers = ::Dalli::ServersArgNormalizer.normalize_servers(servers)

data/lib/dalli/protocol/base.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'forwardable'
+require 'set'
 require 'socket'
 require 'timeout'
 
@@ -24,6 +25,7 @@ module Dalli
         hostname, port, socket_type, @weight, user_creds = ServerConfigParser.parse(attribs)
         @options = client_options.merge(user_creds)
         @raw_mode = client_options[:raw]
+        @verify_reply_keys = client_options[:verify_reply_keys] ? true : false
         @value_marshaller = @raw_mode ? StringMarshaller.new(@options) : ValueMarshaller.new(@options)
         @connection_manager = ConnectionManager.new(hostname, port, socket_type, @options)
       end
@@ -103,6 +105,8 @@ module Dalli
       # Without a block, returns a Hash of { key => [value, cas] }.
       # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       def pipeline_next_responses(&block)
+        return abandon_pipeline if @pipeline_out_of_step
+
         reconnect_on_pipeline_complete!
         values = nil
 
@@ -124,6 +128,13 @@ module Dalli
           # to the noop at the end of the pipeline
           finish_pipeline && break if status && key.nil?
 
+          # A key that wasn't requested means the replies are out of step
+          # (see expect_reply_keys): keep what was read so far, drop the rest
+          if unexpected_reply_key?(key)
+            abandon_pipeline
+            break
+          end
+
           # If the status is ok and the key is not nil, then this is a
           # getkq response with a value that we want to set in the response hash
           unless key.nil?
@@ -252,6 +263,7 @@ module Dalli
         # Use clear (not reset) to keep pipeline_complete? = true, which is
         # the expected state before pipeline_response_setup is called.
         response_buffer.clear
+        expect_reply_keys(keys)
 
         req = +''
         keys.each do |key|
@@ -267,6 +279,7 @@ module Dalli
       def pipelined_get_interleaved(keys, chunk_size, results)
         # Initialize the response buffer for draining during send phase
         response_buffer.ensure_ready
+        expect_reply_keys(keys)
 
         keys.each_slice(chunk_size) do |chunk|
           # Build and write this chunk of requests
@@ -299,6 +312,12 @@ module Dalli
           status, cas, key, value = response_buffer.process_single_getk_response
           break if status.nil? # No complete response available
 
+          # Handled by the next pipeline_next_responses, once sending is done
+          if unexpected_reply_key?(key)
+            @pipeline_out_of_step = true
+            break
+          end
+
           results[key] = [value, cas] unless key.nil?
         end
       rescue SystemCallError, Dalli::NetworkError
@@ -310,6 +329,42 @@ module Dalli
         @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
       end
 
+      # A multi-get only accepts replies for the keys it asked for. memcached
+      # names an item created by auto-create (ma or mg with N) for a
+      # base64-encoded key by its raw bytes, without the b flag; such a key
+      # can contain spaces or CR/LF, which split the reply header and make the
+      # replies after it parse as different ones. A key that wasn't requested
+      # is how that shows up.
+      # Only with the verify_reply_keys option, since building the set costs a
+      # multi-get about 15%. Dalli itself no longer creates such items (see
+      # Meta#decr_incr); the check guards against ones that older versions or
+      # other clients created, on memcached versions with the behavior.
+      def expect_reply_keys(keys)
+        @pipeline_out_of_step = false
+        @expected_reply_keys = (keys.to_set { |k| reply_key_form(k) } if @verify_reply_keys)
+      end
+
+      def unexpected_reply_key?(key)
+        !key.nil? && @expected_reply_keys && !@expected_reply_keys.include?(reply_key_form(key))
+      end
+
+      # Keys compared as bytes; ASCII keys compare equal in any encoding as is
+      def reply_key_form(key)
+        key.ascii_only? ? key : key.b
+      end
+
+      # Keeps what was read, drops the rest of this server's replies, and
+      # closes the connection so nothing unread is left on it. Not a socket
+      # failure, so it doesn't count toward socket_max_failures; the keys not
+      # read are simply missing from the result.
+      def abandon_pipeline
+        Dalli.logger.warn { "#{name} replied to a get_multi with a key that wasn't requested; closing the connection" }
+        @pipeline_out_of_step = false
+        response_buffer.clear
+        @connection_manager.close
+        {}
+      end
+
       # Called after the noop response is received at the end of a set
       # of pipelined gets
       def finish_pipeline

data/lib/dalli/protocol/meta.rb

@@ -213,12 +213,45 @@ module Dalli
       def decr_incr(incr, key, delta, ttl, initial)
         ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
         encoded_key, base64 = KeyRegularizer.encode(key)
+        # memcached stores an item created by ma's auto-create (N/J) for a
+        # base64-encoded key without its binary-key flag, and later replies
+        # name it with the raw key bytes. Create such counters with an add.
+        return decr_incr_with_add(incr, key, delta, ttl, initial) if initial && base64
+
         write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: initial, incr: incr, ttl: ttl,
                                                quiet: quiet?, base64: base64))
         @connection_manager.flush unless quiet?
         response_processor.decr_incr unless quiet?
       end
 
+      # Like decr_incr with an initial value, without memcached's auto-create:
+      # on a miss the counter is added with the initial value (which is
+      # returned, as auto-create would), and the add losing a race to another
+      # client means the counter now exists, so the incr is tried again. In a
+      # quiet block both requests are sent and whichever applies takes effect.
+      def decr_incr_with_add(incr, key, delta, ttl, initial)
+        add_options = { raw: true } # counters are stored raw
+        return quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options) if quiet?
+
+        2.times do
+          value = decr_incr(incr, key, delta, nil, nil)
+          return value unless value.nil?
+
+          write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: false)
+          return initial if response_processor.meta_set_with_cas
+        end
+        decr_incr(incr, key, delta, nil, nil)
+      end
+
+      # rubocop:disable-next Metrics/ParameterLists
+      def quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: nil, incr: incr, ttl: nil,
+                                               quiet: true, base64: base64))
+        # Sent with the rest of the quiet block, by its closing noop
+        write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: true)
+      end
+
       # Other Commands
       def flush(delay = 0)
         write(RequestFormatter.flush(delay: delay))

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '4.3.7'
+  VERSION = '4.3.8'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 4.3.7
+  version: 4.3.8
 platform: ruby
 authors:
 - Peter M. Goldstein
@@ -94,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.21
+rubygems_version: 4.0.23
 specification_version: 4
 summary: High performance memcached client for Ruby
 test_files: []