dalli 5.2.3 → 5.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a31a4159ee5cd637dca6676f343595f653798864bef28b109fbc329debe2aff
-  data.tar.gz: efba0a1a4bb20eff2fd4f6b26c15359351484401502a798fa398b5bf1301d046
+  metadata.gz: 626c8dbb98798247423221395d6d518a87ad9ce64ce268488ecd4eb7d8d7f347
+  data.tar.gz: 999126be42f4c80be3eb566d3f10dfbb8e10b46446aaaf0da546523c0e734eef
 SHA512:
-  metadata.gz: 38f6d3ba18790b94875f18bf6239a520f57144ba763826063471bea2047aa6def2cc528d37de35f29037a0c33ca9bb8b46d7ed0ddc5dccebf8914888c71dafd7
-  data.tar.gz: 9f091240e20313b7cbb9250e900cbf3aa02f9aedbffceabeedc7355bc780cd3895e231c77bb091ab97680dbdac85e093635e3cfbca904fd7f095512181e4a59d
+  metadata.gz: 34aaa230697fd4ac27118ec8a90608d30bdc8dabaf9fc5cfc1b8c4e56a20c701708f1b6b7052fd76c7363f25e238905648296865c41b770f434457005af7aac8
+  data.tar.gz: 7bf1bbe810632bd9733a9cb122900ff9e1b283ade2ef071cec13d609cf4e0b2580de06f1116e9a71685fe98aa7804022c719ef3b921e41e9eb76f2be98b5ab1a

data/CHANGELOG.md

@@ -4,6 +4,22 @@ Dalli Changelog
 Unreleased
 ==========
 
+5.2.4
+==========
+
+Security:
+
+- Keep a connection shared by several threads or fibers in step during a `quiet` block (GHSA-5xvx-r8rg-5qm2)
+  - A `quiet` (or `multi`) block is quiet only for the thread or fiber that runs it. memcached still replies to a quiet request that fails (for example an `add` of a key that exists), and another thread or fiber using the same client while the block was open read that reply as its own, so its later requests could return another key's value. A non-quiet request now reads any pending quiet replies first
+- Don't let a `get_multi` that stopped early leave replies for a later request (GHSA-5xvx-r8rg-5qm2)
+  - A pipelined `get_multi` stopped by a non-network exception (its block raising or breaking, `Timeout.timeout`, or a value that can't be deserialized) left its unread replies in the response buffer, and a later `get_multi` large enough to use the interleaved path returned them as its own. The buffer is now cleared when a connection closes or a pipelined get starts, and unfinished pipelines are aborted
+  - Affects every supported line (the `get_multi` issue, 4.3 and later); fixed in 5.2.4, 5.1.6, 5.0.12, 4.3.9 and 3.2.15. On 5.2, clients with `defer_drain: true` weren't affected by the `quiet` issue
+
+Bug fixes:
+
+- A failed TLS handshake is now handled as a network error even when openssl wasn't loaded before Dalli
+  - The fix in 5.2.3 (#1215) took effect only when the application had already required openssl: otherwise Dalli's list of TLS errors was empty, so a failed handshake still raised `OpenSSL::SSL::SSLError` and left the socket open
+
 5.2.3
 ==========
 

data/lib/dalli/client.rb

@@ -434,6 +434,11 @@ module Dalli
     # the defer_drain option, before the next non-quiet request), and
     # Dalli client methods invoked inside the block will not
     # have return values
+    #
+    # The block is quiet only for the thread or fiber that runs it. Another
+    # thread or fiber sharing this client can make requests meanwhile; a
+    # non-quiet one first sends the block's buffered requests and discards
+    # their replies, so it reads only its own.
     def quiet
       old = Thread.current[::Dalli::QUIET]
       Thread.current[::Dalli::QUIET] = true

data/lib/dalli/options.rb

@@ -43,7 +43,7 @@ module Dalli
       end
     end
 
-    def pipeline_abort
+    def pipeline_abort(...)
       @lock.synchronize do
         super
       end

data/lib/dalli/pipelined_getter.rb

@@ -32,15 +32,21 @@ module Dalli
       return {} if keys.empty?
 
       @ring.lock do
-        # Stores partial results collected during interleaved send phase
-        @partial_results = {}
-        servers = setup_requests(keys, req_options, return_cas: return_cas)
-        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
-        # First yield any partial results collected during interleaved send
-        yield_partial_results(&block)
-
-        servers = fetch_responses(servers, start_time, @ring.socket_timeout, &block) until servers.empty?
+        # The servers with replies still to read, updated in place
+        servers = []
+        begin
+          # Stores partial results collected during interleaved send phase
+          @partial_results = {}
+          setup_requests(servers, keys, req_options, return_cas: return_cas)
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          # First yield any partial results collected during interleaved send
+          yield_partial_results(&block)
+
+          fetch_responses(servers, start_time, @ring.socket_timeout, &block) until servers.empty?
+        ensure
+          abort_unfinished(servers)
+        end
       end
     rescue Dalli::RetryableNetworkError => e
       Dalli.logger.debug { e.inspect }
@@ -96,9 +102,9 @@ module Dalli
 
     # Sends each server's queries and its terminating noop before building
     # the next server's, so memcached is already answering the first servers
-    # while the rest are prepared. Returns the servers with a pending response.
-    def setup_requests(keys, req_options = nil, return_cas: true)
-      started = []
+    # while the rest are prepared. Adds the servers with a pending response
+    # to started.
+    def setup_requests(started, keys, req_options = nil, return_cas: true)
       groups_for_keys(keys).each do |server, keys_for_server|
         make_getkq_request(server, keys_for_server, req_options, return_cas: return_cas)
         next unless server.connected?
@@ -111,7 +117,6 @@ module Dalli
       rescue Dalli::DalliError
         started.delete(server)
       end
-      started
     end
 
     ##
@@ -155,29 +160,39 @@ module Dalli
       raise
     end
 
-    # Swallows Dalli::NetworkError
+    # Swallows Dalli::NetworkError. Empties servers, so they aren't aborted
+    # again.
     def abort_without_timeout(servers)
       servers.each(&:pipeline_abort)
+      servers.clear
+    end
+
+    # When #process stops before reading every server's replies (the caller's
+    # block raises or breaks, a Timeout.timeout fires, a value can't be
+    # deserialized), the rest are still on the socket and some may be in the
+    # response buffer. Closing those connections keeps them from being read
+    # as a later request's replies. Aborts that #process already handled
+    # emptied servers.
+    def abort_unfinished(servers)
+      servers.each { |server| server.pipeline_abort(timed_out: false) }
+      servers.clear
     end
 
+    # Reads the replies available, removing finished servers from servers
     def fetch_responses(servers, start_time, timeout, &block)
       # Remove any servers which are not connected
       servers.select!(&:connected?)
-      return [] if servers.empty?
+      return if servers.empty?
 
       time_left = remaining_time(start_time, timeout)
       readable_servers = servers_with_response(servers, time_left)
-      if readable_servers.empty?
-        abort_with_timeout(servers)
-        return []
-      end
+      return abort_with_timeout(servers) if readable_servers.empty?
 
       # Loop through the servers with responses, and
       # delete any from our list that are finished
       readable_servers.each do |server|
         servers.delete(server) if process_server(server, &block)
       end
-      servers
     rescue NetworkError
       # Abort and raise if we encountered a network error.  This triggers
       # a retry at the top level on RetryableNetworkError.
@@ -194,12 +209,10 @@ module Dalli
 
     # Swallows Dalli::NetworkError
     def abort_with_timeout(servers)
-      abort_without_timeout(servers)
       servers.each do |server|
         Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
       end
-
-      true # Required to simplify caller
+      abort_without_timeout(servers)
     end
 
     # Processes responses from a server.  Returns true if there are no

data/lib/dalli/protocol/base.rb

@@ -69,7 +69,7 @@ module Dalli
         begin
           request_completed = false
           @connection_manager.start_request!
-          drain_deferred_quiet_responses(opkey)
+          drain_pending_quiet_responses(opkey)
           response = send(opkey, *args)
           # Quiet requests can leave replies on the socket; remember that so the
           # end of the quiet block knows this server needs draining
@@ -147,7 +147,7 @@ module Dalli
         verify_pipelined_state(:getkq)
         write_noop
         # Use ensure_ready instead of reset to preserve any data already buffered
-        # during interleaved pipelined get draining
+        # while this get_multi's interleaved requests were sent
         response_buffer.ensure_ready
       end
 
@@ -201,6 +201,10 @@ module Dalli
             end
           end
 
+          # A request to this server from inside the block closes the
+          # connection, and with it the rest of this get_multi's replies
+          raise_pipeline_closed unless response_buffer.in_progress?
+
           # Get the next response from the buffer
           status, cas, key, value = response_buffer.process_single_getk_response
         end
@@ -210,21 +214,24 @@ module Dalli
       # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
       # Abort current pipelined get. Generally used to signal an external
-      # timeout during pipelined get.  The underlying socket is
+      # timeout during pipelined get, or that the get_multi stopped before
+      # reading all its replies (timed_out: false).  The underlying socket is
       # disconnected, and the exception is swallowed.
       #
       # Returns nothing.
-      def pipeline_abort
+      def pipeline_abort(timed_out: true)
         response_buffer.clear
+        @pipeline_out_of_step = false
+        @expected_reply_keys = nil
         @connection_manager.abort_request!
         return true unless connected?
 
         # Closes the connection, which ensures that our connection is in a
         # clean state for future requests. The get_multi ran out of time
-        # overall rather than hitting a socket error, so it doesn't count
-        # toward socket_max_failures: two slow get_multis in a row would
-        # otherwise mark a healthy server down.
-        Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
+        # overall, or stopped early, rather than hitting a socket error, so it
+        # doesn't count toward socket_max_failures: two slow get_multis in a
+        # row would otherwise mark a healthy server down.
+        log_pipeline_abort(timed_out)
         @connection_manager.close
         true
       end
@@ -256,10 +263,15 @@ module Dalli
       ALLOWED_QUIET_OPS = %i[add replace set delete incr decr append prepend flush noop].freeze
       private_constant :ALLOWED_QUIET_OPS
 
-      # With defer_drain, quiet blocks leave their replies unread. Read them
-      # before any non-quiet request, which reads a reply of its own.
-      def drain_deferred_quiet_responses(opkey)
-        return unless @defer_drain && !quiet? && opkey != :noop && @connection_manager.quiet_responses_pending?
+      # Reads the replies still pending from quiet requests before any
+      # non-quiet request, which reads a reply of its own. With defer_drain,
+      # quiet blocks leave their replies unread. Without it, a quiet block
+      # reads them at its end, but the connection is shared: another thread or
+      # fiber (quiet? is fiber-local) can make a request while the block is
+      # still open, and would otherwise read the block's replies as its own.
+      # The noop also sends any quiet requests still buffered.
+      def drain_pending_quiet_responses(opkey)
+        return unless @connection_manager.quiet_responses_pending? && opkey != :noop && !quiet?
 
         noop
       end
@@ -372,8 +384,9 @@ module Dalli
       # This prevents socket buffer deadlock when sending many keys.
       # Populates the provided results hash with any responses drained during send.
       def pipelined_get_interleaved(keys, chunk_size, results, options = nil)
-        # Initialize the response buffer for draining during send phase
-        response_buffer.ensure_ready
+        # Start with an empty buffer for draining during the send phase. Bytes
+        # left by an earlier get_multi that stopped early belong to its replies.
+        response_buffer.reset
         response_buffer.raw = raw_request?(options)
         expect_reply_keys(keys)
 
@@ -421,8 +434,12 @@ module Dalli
         nil
       end
 
+      # Registered with the connection manager, which clears it whenever the
+      # connection closes, so a new connection never parses an old one's bytes
       def response_buffer
-        @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
+        @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor).tap do |buffer|
+          @connection_manager.response_buffer = buffer
+        end
       end
 
       # A multi-get only accepts replies for the keys it asked for. memcached
@@ -474,6 +491,19 @@ module Dalli
         @connection_manager.reconnect! 'pipelined get has completed' if pipeline_complete?
       end
 
+      def raise_pipeline_closed
+        raise Dalli::DalliError, "The connection to #{name} closed while a get_multi was reading its replies " \
+                                 '(a request made from inside the get_multi block?)'
+      end
+
+      def log_pipeline_abort(timed_out)
+        if timed_out
+          Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
+        else
+          Dalli.logger.debug { "#{name} get_multi stopped before reading all replies; closing the connection" }
+        end
+      end
+
       # The single-server set_multi path passes its whole pairs hash as the
       # first argument; log only its keys, never the values
       def log_marshal_err(key, err)

data/lib/dalli/protocol/connection_manager.rb

@@ -26,6 +26,9 @@ module Dalli
 
       attr_accessor :hostname, :port, :socket_type, :options
       attr_reader :sock
+      # The buffer replies are read into (see Base#response_buffer). It's
+      # cleared with the write buffer whenever the connection closes.
+      attr_writer :response_buffer
 
       def initialize(hostname, port, socket_type, client_options)
         @hostname = hostname
@@ -38,6 +41,7 @@ module Dalli
         @pid = nil
         @write_buffer = []
         @write_buffer_bytes = 0
+        @response_buffer = nil
 
         @fail_count = 0
         reset_down_info
@@ -133,6 +137,7 @@ module Dalli
           @pid = nil
           @quiet_responses_pending = false
           discard_write_buffer
+          @response_buffer&.clear
           abort_request!
         end
       end

data/lib/dalli/protocol/meta.rb

@@ -283,10 +283,12 @@ module Dalli
       # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
       # We need to read all the responses at once.
       # A write that expects a reply is flushed so the reply can be read. A write
-      # in a quiet block stays buffered until the block's noop, unless
-      # defer_drain is on: then no noop follows at the end of the block, so it is
-      # flushed right away. (pipelined_set passes quiet: true outside a quiet
-      # block; its caller flushes with a terminating noop.)
+      # in a quiet block stays buffered until a noop: the one at the end of the
+      # block, or the one another thread or fiber sends to drain the block's
+      # replies before its own request. With defer_drain no noop follows at the
+      # end of the block, so it is flushed right away. (pipelined_set passes
+      # quiet: true outside a quiet block; its caller flushes with a
+      # terminating noop.)
       def finish_write(quiet)
         @connection_manager.flush if !quiet || (@defer_drain && quiet?)
       end

data/lib/dalli/protocol.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'openssl'
 require 'timeout'
 
 module Dalli
@@ -19,11 +20,8 @@ module Dalli
     # SSL errors during the handshake or during read/write operations count as
     # a failed request: they trigger reconnection and, after
     # socket_max_failures, mark the server down.
-    SSL_ERRORS =
-      if defined?(OpenSSL::SSL::SSLError)
-        [OpenSSL::SSL::SSLError]
-      else
-        []
-      end
+    # openssl is required above: this file loads before dalli/socket, so a
+    # defined? check here would find OpenSSL missing and leave this empty.
+    SSL_ERRORS = [OpenSSL::SSL::SSLError].freeze
   end
 end

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '5.2.3'
+  VERSION = '5.2.4'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.6.27'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 5.2.3
+  version: 5.2.4
 platform: ruby
 authors:
 - Peter M. Goldstein