dalli 3.2.13 → 3.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f7bcbe37e631c3c733d1487e519501497702b873e81e6588d6dfad1ab96a65b
-  data.tar.gz: d7a953d2c0052334d8edb7fa45306724ffa25cec64436dcceccc9847d057e8bd
+  metadata.gz: 5b52d92d26a7277b6f65b402c082be2f99b98ca9cb7596af709ac70848167a53
+  data.tar.gz: 2cc59e69e536f07ec79ec6e37ffcae997c1e47231aaa213e32fdce02b3b3e123
 SHA512:
-  metadata.gz: 8d7b0ed94102d333f611736c48d944b0f1010760ae646003db60cd02c90b1a047a5fd10f7a5ef077527a88594f8844cbe08f9647259749e78ade74d09d2e851b
-  data.tar.gz: bf86e4604fc18eaf2bdf57d6abad82258745238112a9866e5da9e05c0f45b44d62f82ebe8b5519ecc83d00ec6ee94e2d11bb931aa14171081a0b731745633e6d
+  metadata.gz: 8db03f1265942f604b68bdded36f70aa00e859269950367df91c62ef1b2d1011ba6166269125f4ab377c4b4acd4acfd763c3f17a867d2e357479b259c0592ffc
+  data.tar.gz: 7f15a910062c45a776e3bc3964a3555b4a3c0c0a5897877dc170ee5a80d16a313bb4eb0da64821ca14d930ffbea1445855e2f1ca7fe2a068a33765aabc064636

data/CHANGELOG.md

@@ -4,6 +4,15 @@ Dalli Changelog
 Unreleased
 ==========
 
+3.2.14
+==========
+
+- Don't rely on memcached's auto-create for keys that need base64 encoding (GHSA-r47x-6p6r-mvcp)
+  - memcached stores an item made by auto-create (`ma` with `N`/`J`, or `mg` with `N`) for a base64-encoded key without its binary-key flag, and later replies name the item by its raw key bytes. A key containing spaces or CR/LF could then make a multi-get read the replies after it as different ones
+  - `incr` and `decr` with an initial value now create such counters with an add instead. Return values are unchanged
+  - The new `verify_reply_keys` option makes `get_multi` ignore replies for keys they didn't request, which guards against items created by older Dalli versions or other clients until memcached is upgraded. It's off by default, since it costs a multi-get about 15%
+  - Affects `protocol: :meta` since 3.2.0; fixed in 5.2.3, 5.1.5, 5.0.11, 4.3.8 and 3.2.14. memcached 1.6.45 and earlier still name these items by their raw key bytes
+
 3.2.13
 ==========
 

data/README.md

@@ -37,6 +37,10 @@ Dalli::Client.new('localhost:11211', serializer: Dalli::JSONSerializer)
 
 Per-request `raw: true` returns the stored bytes without deserializing or decompressing them, on every read method that accepts it (`get` and `fetch`).
 
+### Counters and verify_reply_keys
+
+memcached names an item made by auto-create for a base64-encoded key (one with whitespace, control or non-ASCII characters) by its raw key bytes in later replies (with `protocol: :meta`). Since 3.2.14, Dalli creates counters for such keys with an add rather than auto-create. Items created before then, or by other clients, can still be named that way until memcached releases a fix (1.6.45 and earlier behave this way). If keys passed to `incr` or `decr` with an initial value can contain untrusted input, set `verify_reply_keys: true` until then: multi-gets then ignore replies for keys they didn't request, at a cost of about 15% per multi-get.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/dalli/client.rb

@@ -45,6 +45,11 @@ module Dalli
     #                   useful for injecting a FIPS compliant hash object.
     # - :protocol - one of either :binary or :meta, defaulting to :binary.  This sets the protocol that Dalli uses
     #               to communicate with memcached.
+    # - :verify_reply_keys - defaults to false. When true, get_multi only accepts replies for the keys it requested;
+    #                        after a reply naming another key, the rest of that server's replies are treated as misses.
+    #                        With the meta protocol, memcached can name an item by its raw key bytes when the item was
+    #                        created by auto-create from a base64-encoded key, which Dalli no longer does. Costs a
+    #                        multi-get about 15%.
     #
     def initialize(servers = nil, options = {})
       @normalized_servers = ::Dalli::ServersArgNormalizer.normalize_servers(servers)

data/lib/dalli/protocol/base.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'forwardable'
+require 'set'
 require 'socket'
 require 'timeout'
 
@@ -23,6 +24,7 @@ module Dalli
       def initialize(attribs, client_options = {})
         hostname, port, socket_type, @weight, user_creds = ServerConfigParser.parse(attribs)
         @options = client_options.merge(user_creds)
+        @verify_reply_keys = client_options[:verify_reply_keys] ? true : false
         @value_marshaller = ValueMarshaller.new(@options)
         @connection_manager = ConnectionManager.new(hostname, port, socket_type, @options)
       end
@@ -105,7 +107,7 @@ module Dalli
 
           # If the status is ok and the key is not nil, then this is a
           # getkq response with a value that we want to set in the response hash
-          values[key] = [value, cas] unless key.nil?
+          break unless keep_reading?(values, key, value, cas)
 
           # Get the next response from the buffer
           status, cas, key, value = response_buffer.process_single_getk_response
@@ -220,6 +222,7 @@ module Dalli
       end
 
       def pipelined_get(keys)
+        expect_reply_keys(keys)
         req = +''
         keys.each do |key|
           req << quiet_get_request(key)
@@ -232,6 +235,50 @@ module Dalli
         @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
       end
 
+      # A multi-get only accepts replies for the keys it asked for. With the
+      # meta protocol, memcached names an item created by auto-create (ma or mg
+      # with N) for a base64-encoded key by its raw bytes, without the b flag;
+      # such a key can contain spaces or CR/LF, which split the reply header and
+      # make the replies after it parse as different ones. A key that wasn't
+      # requested is how that shows up. Only with the verify_reply_keys option,
+      # since building the set costs a multi-get about 15%.
+      def expect_reply_keys(keys)
+        @expected_reply_keys = (keys.to_set { |k| reply_key_form(k) } if @verify_reply_keys)
+      end
+
+      # Stores a getkq reply's value and returns true, unless its key wasn't
+      # requested: then the replies are out of step, so what was read so far is
+      # kept, the rest dropped, and false returned to stop reading
+      def keep_reading?(values, key, value, cas)
+        return true if key.nil?
+
+        if unexpected_reply_key?(key)
+          abandon_pipeline
+          return false
+        end
+
+        values[key] = [value, cas]
+        true
+      end
+
+      def unexpected_reply_key?(key)
+        !key.nil? && @expected_reply_keys && !@expected_reply_keys.include?(reply_key_form(key))
+      end
+
+      # Keys compared as bytes; ASCII keys compare equal in any encoding as is
+      def reply_key_form(key)
+        key.ascii_only? ? key : key.b
+      end
+
+      # Drops the rest of this server's replies and closes the connection so
+      # nothing unread is left on it. Not a socket failure, so it doesn't count
+      # toward socket_max_failures; the keys not read are simply missing.
+      def abandon_pipeline
+        Dalli.logger.warn { "#{name} replied to a get_multi with a key that wasn't requested; closing the connection" }
+        response_buffer.clear
+        @connection_manager.close
+      end
+
       # Called after the noop response is received at the end of a set
       # of pipelined gets
       def finish_pipeline

data/lib/dalli/protocol/meta.rb

@@ -13,6 +13,7 @@ module Dalli
     ##
     class Meta < Base
       TERMINATOR = "\r\n"
+      COUNTER_ADD_OPTIONS = { raw: true }.freeze # counters are stored raw
 
       def response_processor
         @response_processor ||= ResponseProcessor.new(@connection_manager, @value_marshaller)
@@ -129,11 +130,42 @@ module Dalli
       def decr_incr(incr, key, delta, ttl, initial)
         ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
         encoded_key, base64 = KeyRegularizer.encode(key)
+        # memcached stores an item created by ma's auto-create (N/J) for a
+        # base64-encoded key without its binary-key flag, and later replies
+        # name it with the raw key bytes. Create such counters with an add.
+        return decr_incr_with_add(incr, key, delta, ttl, initial) if initial && base64
+
         write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: initial, incr: incr, ttl: ttl,
                                                quiet: quiet?, base64: base64))
         response_processor.decr_incr unless quiet?
       end
 
+      # Like decr_incr with an initial value, without memcached's auto-create:
+      # on a miss the counter is added with the initial value (which is
+      # returned, as auto-create would), and the add losing a race to another
+      # client means the counter now exists, so the incr is tried again. In a
+      # quiet block both requests are sent and whichever applies takes effect.
+      def decr_incr_with_add(incr, key, delta, ttl, initial)
+        return quiet_decr_incr_with_add(incr, key, delta, ttl, initial) if quiet?
+
+        2.times do
+          value = decr_incr(incr, key, delta, nil, nil)
+          return value unless value.nil?
+
+          write_storage_req(:add, key, initial.to_s, ttl, nil, COUNTER_ADD_OPTIONS)
+          return initial if response_processor.meta_set_with_cas
+        end
+        decr_incr(incr, key, delta, nil, nil)
+      end
+
+      def quiet_decr_incr_with_add(incr, key, delta, ttl, initial)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: nil, incr: incr, ttl: nil,
+                                               quiet: true, base64: base64))
+        # Sent with the rest of the quiet block, by its closing noop
+        write_storage_req(:add, key, initial.to_s, ttl, nil, COUNTER_ADD_OPTIONS)
+      end
+
       # Other Commands
       def flush(delay = 0)
         write(RequestFormatter.flush(delay: delay))

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '3.2.13'
+  VERSION = '3.2.14'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 3.2.13
+  version: 3.2.14
 platform: ruby
 authors:
 - Peter M. Goldstein
@@ -90,7 +90,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.21
+rubygems_version: 4.0.23
 specification_version: 4
 summary: High performance memcached client for Ruby
 test_files: []