dalli 4.3.10 → 5.0.0

data/lib/dalli/protocol/base.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'forwardable'
-require 'set'
 require 'socket'
 require 'timeout'
 
@@ -23,9 +22,9 @@ module Dalli
 
       def initialize(attribs, client_options = {})
         hostname, port, socket_type, @weight, user_creds = ServerConfigParser.parse(attribs)
+        warn_uri_credentials(user_creds)
         @options = client_options.merge(user_creds)
         @raw_mode = client_options[:raw]
-        @verify_reply_keys = client_options[:verify_reply_keys] ? true : false
         @value_marshaller = @raw_mode ? StringMarshaller.new(@options) : ValueMarshaller.new(@options)
         @connection_manager = ConnectionManager.new(hostname, port, socket_type, @options)
       end
@@ -36,36 +35,20 @@ module Dalli
         @raw_mode
       end
 
-      # True when values should come back as stored: the client is in raw mode,
-      # or this request passed raw: true. A stored value is then never
-      # deserialized or decompressed for a caller who asked for raw bytes.
-      def raw_request?(options)
-        raw_mode? || (options.is_a?(Hash) && options[:raw]) ? true : false
-      end
-
       # Chokepoint method for error handling and ensuring liveness
       def request(opkey, *args)
         verify_state(opkey)
 
         begin
           @connection_manager.start_request!
-          drain_pending_quiet_responses(opkey)
           response = send(opkey, *args)
-          # Quiet requests can leave replies on the socket; remember that so a
-          # later non-quiet request knows to read them first
-          @connection_manager.mark_quiet_responses_pending! if quiet? && opkey != :noop
 
           # pipelined_get/pipelined_get_interleaved emit query but don't read the response(s)
           @connection_manager.finish_request! unless %i[pipelined_get pipelined_get_interleaved].include?(opkey)
 
           response
-        rescue Dalli::MarshalError, Dalli::ValueOverMaxSize => e
-          log_marshal_err(args.first, e) if e.is_a?(Dalli::MarshalError)
-          # Raised only while the connection is clean: a value is marshalled
-          # before any of its request is written. Leaving the request in
-          # progress would close the connection on the next request, which
-          # discards earlier quiet writes still waiting in the write buffer.
-          @connection_manager.abort_request!
+        rescue Dalli::MarshalError => e
+          log_marshal_err(args.first, e)
           raise
         rescue Dalli::DalliError
           raise
@@ -100,7 +83,7 @@ module Dalli
         verify_pipelined_state(:getkq)
         write_noop
         # Use ensure_ready instead of reset to preserve any data already buffered
-        # while this get_multi's interleaved requests were sent
+        # during interleaved pipelined get draining
         response_buffer.ensure_ready
       end
 
@@ -109,25 +92,12 @@ module Dalli
       # repeatedly whenever this server's socket is readable until
       # #pipeline_complete?.
       #
-      # When a block is given, yields (key, value, cas) for each response,
-      # avoiding intermediate Hash allocation. Returns nil.
-      # Without a block, returns a Hash of { key => [value, cas] }.
-      # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      def pipeline_next_responses(&block)
-        return abandon_pipeline if @pipeline_out_of_step
-
+      # Returns a Hash of kv pairs received.
+      def pipeline_next_responses
         reconnect_on_pipeline_complete!
-        values = nil
+        values = {}
 
-        # Only the read is socket I/O. The caller's block runs below, outside
-        # this rescue, so an Errno or Timeout::Error it raises reaches the
-        # caller instead of being mistaken for a network failure (which closed
-        # the connection and retried the whole get_multi, yielding keys twice).
-        begin
-          response_buffer.read
-        rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
-          @connection_manager.error_on_request!(e)
-        end
+        response_buffer.read
 
         status, cas, key, value = response_buffer.process_single_getk_response
         # status is not nil only if we have a full response to parse
@@ -137,55 +107,33 @@ module Dalli
           # to the noop at the end of the pipeline
           finish_pipeline && break if status && key.nil?
 
-          # A key that wasn't requested means the replies are out of step
-          # (see expect_reply_keys): keep what was read so far, drop the rest
-          if unexpected_reply_key?(key)
-            abandon_pipeline
-            break
-          end
-
           # If the status is ok and the key is not nil, then this is a
           # getkq response with a value that we want to set in the response hash
-          unless key.nil?
-            if block
-              yield key, value, cas
-            else
-              values ||= {}
-              values[key] = [value, cas]
-            end
-          end
-
-          # A request to this server from inside the block closes the
-          # connection, and with it the rest of this get_multi's replies
-          raise_pipeline_closed unless response_buffer.in_progress?
+          values[key] = [value, cas] unless key.nil?
 
           # Get the next response from the buffer
           status, cas, key, value = response_buffer.process_single_getk_response
         end
 
-        values || {}
+        values
+      rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
+        @connection_manager.error_on_request!(e)
       end
 
       # Abort current pipelined get. Generally used to signal an external
-      # timeout during pipelined get, or that the get_multi stopped before
-      # reading all its replies (timed_out: false).  The underlying socket is
+      # timeout during pipelined get.  The underlying socket is
       # disconnected, and the exception is swallowed.
       #
       # Returns nothing.
-      def pipeline_abort(timed_out: true)
+      def pipeline_abort
         response_buffer.clear
-        @pipeline_out_of_step = false
-        @expected_reply_keys = nil
         @connection_manager.abort_request!
         return true unless connected?
 
-        # Closes the connection, which ensures that our connection is in a
-        # clean state for future requests. The get_multi ran out of time
-        # overall, or stopped early, rather than hitting a socket error, so it
-        # doesn't count toward socket_max_failures: two slow get_multis in a
-        # row would otherwise mark a healthy server down.
-        log_pipeline_abort(timed_out)
-        @connection_manager.close
+        # Closes the connection, which ensures that our connection
+        # is in a clean state for future requests
+        @connection_manager.error_on_request!('External timeout')
+      rescue NetworkError
         true
       end
 
@@ -194,18 +142,6 @@ module Dalli
         !response_buffer.in_progress?
       end
 
-      def username
-        @options[:username] || ENV.fetch('MEMCACHE_USERNAME', nil)
-      end
-
-      def password
-        @options[:password] || ENV.fetch('MEMCACHE_PASSWORD', nil)
-      end
-
-      def require_auth?
-        !username.nil?
-      end
-
       def quiet?
         Thread.current[::Dalli::QUIET]
       end
@@ -215,21 +151,19 @@ module Dalli
 
       private
 
-      ALLOWED_QUIET_OPS = %i[add replace set delete incr decr append prepend flush noop].freeze
-      private_constant :ALLOWED_QUIET_OPS
+      URI_CREDENTIAL_WARNING = 'Dalli 5.0 removed SASL authentication. ' \
+                               'Credentials in memcached:// URIs are ignored.'
+      private_constant :URI_CREDENTIAL_WARNING
 
-      # Reads the replies still pending from quiet requests before any
-      # non-quiet request, which reads a reply of its own. A quiet block reads
-      # them at its end, but the connection is shared: another thread or fiber
-      # (quiet? is fiber-local) can make a request while the block is still
-      # open, and would otherwise read the block's replies as its own. The
-      # noop also sends any quiet requests still buffered.
-      def drain_pending_quiet_responses(opkey)
-        return unless @connection_manager.quiet_responses_pending? && opkey != :noop && !quiet?
+      def warn_uri_credentials(user_creds)
+        return if user_creds[:username].nil? && user_creds[:password].nil?
 
-        noop
+        Dalli.logger.warn(URI_CREDENTIAL_WARNING)
       end
 
+      ALLOWED_QUIET_OPS = %i[add replace set delete incr decr append prepend flush noop].freeze
+      private_constant :ALLOWED_QUIET_OPS
+
       def verify_allowed_quiet!(opkey)
         return if ALLOWED_QUIET_OPS.include?(opkey)
 
@@ -281,8 +215,7 @@ module Dalli
 
       def connect
         @connection_manager.establish_connection
-        authenticate_connection if require_auth?
-        @version = version # Connect socket if not authed
+        @version = version
         up!
       end
 
@@ -291,7 +224,6 @@ module Dalli
         # Use clear (not reset) to keep pipeline_complete? = true, which is
         # the expected state before pipeline_response_setup is called.
         response_buffer.clear
-        expect_reply_keys(keys)
 
         req = +''
         keys.each do |key|
@@ -305,10 +237,8 @@ module Dalli
       # This prevents socket buffer deadlock when sending many keys.
       # Populates the provided results hash with any responses drained during send.
       def pipelined_get_interleaved(keys, chunk_size, results)
-        # Start with an empty buffer for draining during the send phase. Bytes
-        # left by an earlier get_multi that stopped early belong to its replies.
-        response_buffer.reset
-        expect_reply_keys(keys)
+        # Initialize the response buffer for draining during send phase
+        response_buffer.ensure_ready
 
         keys.each_slice(chunk_size) do |chunk|
           # Build and write this chunk of requests
@@ -341,12 +271,6 @@ module Dalli
           status, cas, key, value = response_buffer.process_single_getk_response
           break if status.nil? # No complete response available
 
-          # Handled by the next pipeline_next_responses, once sending is done
-          if unexpected_reply_key?(key)
-            @pipeline_out_of_step = true
-            break
-          end
-
           results[key] = [value, cas] unless key.nil?
         end
       rescue SystemCallError, Dalli::NetworkError
@@ -354,48 +278,8 @@ module Dalli
         nil
       end
 
-      # Registered with the connection manager, which clears it whenever the
-      # connection closes, so a new connection never parses an old one's bytes
       def response_buffer
-        @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor).tap do |buffer|
-          @connection_manager.response_buffer = buffer
-        end
-      end
-
-      # A multi-get only accepts replies for the keys it asked for. memcached
-      # names an item created by auto-create (ma or mg with N) for a
-      # base64-encoded key by its raw bytes, without the b flag; such a key
-      # can contain spaces or CR/LF, which split the reply header and make the
-      # replies after it parse as different ones. A key that wasn't requested
-      # is how that shows up.
-      # Only with the verify_reply_keys option, since building the set costs a
-      # multi-get about 15%. Dalli itself no longer creates such items (see
-      # Meta#decr_incr); the check guards against ones that older versions or
-      # other clients created, on memcached versions with the behavior.
-      def expect_reply_keys(keys)
-        @pipeline_out_of_step = false
-        @expected_reply_keys = (keys.to_set { |k| reply_key_form(k) } if @verify_reply_keys)
-      end
-
-      def unexpected_reply_key?(key)
-        !key.nil? && @expected_reply_keys && !@expected_reply_keys.include?(reply_key_form(key))
-      end
-
-      # Keys compared as bytes; ASCII keys compare equal in any encoding as is
-      def reply_key_form(key)
-        key.ascii_only? ? key : key.b
-      end
-
-      # Keeps what was read, drops the rest of this server's replies, and
-      # closes the connection so nothing unread is left on it. Not a socket
-      # failure, so it doesn't count toward socket_max_failures; the keys not
-      # read are simply missing from the result.
-      def abandon_pipeline
-        Dalli.logger.warn { "#{name} replied to a get_multi with a key that wasn't requested; closing the connection" }
-        @pipeline_out_of_step = false
-        response_buffer.clear
-        @connection_manager.close
-        {}
+        @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
       end
 
       # Called after the noop response is received at the end of a set
@@ -411,19 +295,6 @@ module Dalli
         @connection_manager.reconnect! 'pipelined get has completed' if pipeline_complete?
       end
 
-      def raise_pipeline_closed
-        raise Dalli::DalliError, "The connection to #{name} closed while a get_multi was reading its replies " \
-                                 '(a request made from inside the get_multi block?)'
-      end
-
-      def log_pipeline_abort(timed_out)
-        if timed_out
-          Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
-        else
-          Dalli.logger.debug { "#{name} get_multi stopped before reading all replies; closing the connection" }
-        end
-      end
-
       def log_marshal_err(key, err)
         Dalli.logger.error "Marshalling error for key '#{key}': #{err.message}"
         Dalli.logger.error 'You are trying to cache a Ruby object which cannot be serialized to memcached.'

data/lib/dalli/protocol/connection_manager.rb

@@ -28,9 +28,6 @@ module Dalli
 
       attr_accessor :hostname, :port, :socket_type, :options
       attr_reader :sock
-      # The buffer replies are read into (see Base#response_buffer). It's
-      # cleared with the write buffer whenever the connection closes.
-      attr_writer :response_buffer
 
       def initialize(hostname, port, socket_type, client_options)
         @hostname = hostname
@@ -38,14 +35,9 @@ module Dalli
         @socket_type = socket_type
         @options = DEFAULTS.merge(client_options)
         @request_in_progress = false
-        @quiet_responses_pending = false
         @sock = nil
         @pid = nil
-        @write_buffer = []
-        @write_buffer_bytes = 0
-        @response_buffer = nil
 
-        @fail_count = 0
         reset_down_info
       end
 
@@ -61,13 +53,11 @@ module Dalli
         Dalli.logger.debug { "Dalli::Server#connect #{name}" }
 
         @sock = memcached_socket
-        # Writes are buffered in @write_buffer instead; see WRITE_BUFFER_FLUSH_BYTES
-        @sock.sync = true
-        @pid = Process.pid
+        @sock.sync = false # Enable buffered I/O for better performance
+        @pid = PIDCache.pid
         @request_in_progress = false
-        @quiet_responses_pending = false
-      rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError, SocketError => e
-        # SocketError = DNS resolution failure; SSL_ERRORS = failed TLS handshake
+      rescue SystemCallError, *TIMEOUT_ERRORS, EOFError, SocketError => e
+        # SocketError = DNS resolution failure
         error_on_request!(e)
       end
 
@@ -95,8 +85,6 @@ module Dalli
       def down!
         close
         log_down_detected
-        # Once down_retry_delay passes, the server gets a full set of attempts
-        @fail_count = 0
 
         @error = $ERROR_INFO&.class&.name
         @msg ||= $ERROR_INFO&.message
@@ -126,44 +114,19 @@ module Dalli
         return unless @sock
 
         begin
-          close_socket
+          @sock.close
         rescue StandardError
           nil
         end
         @sock = nil
         @pid = nil
-        @quiet_responses_pending = false
-        discard_write_buffer
-        @response_buffer&.clear
         abort_request!
       end
 
-      # A forked child shares the parent's connection, so it closes only its
-      # own file descriptor. Closing the TLS socket itself would send
-      # close_notify and end the parent's TLS session.
-      def close_socket
-        fork_detected? ? @sock.to_io.close : @sock.close
-      end
-
       def connected?
         !@sock.nil?
       end
 
-      # True when a quiet request was written on this connection and its
-      # replies (memcached only suppresses some of them) haven't been
-      # consumed by a noop yet.
-      def quiet_responses_pending?
-        @quiet_responses_pending
-      end
-
-      def mark_quiet_responses_pending!
-        @quiet_responses_pending = true
-      end
-
-      def clear_quiet_responses_pending!
-        @quiet_responses_pending = false
-      end
-
       def request_in_progress?
         @request_in_progress
       end
@@ -178,88 +141,47 @@ module Dalli
         raise '[Dalli] No request in progress. This may be a bug in Dalli.' unless @request_in_progress
 
         @request_in_progress = false
-        # A completed request is what proves the server healthy again, so the
-        # failure count resets here rather than on reconnect: a server that
-        # accepts connections but never answers would otherwise reset it on
-        # every retry and never reach socket_max_failures.
-        @fail_count = 0
       end
 
       def abort_request!
         @request_in_progress = false
       end
 
-      # The longest reply line Dalli reads, in bytes, CRLF included. A meta
-      # reply header is at most about 500 bytes: "VA <size>" plus its flags,
-      # the largest being a 250-byte key base64-encoded to 336 bytes, a
-      # 20-digit CAS, a 32-byte opaque and a few numeric fields. The longest
-      # stats line (stats settings) carries a file path such as ext_path, up
-      # to PATH_MAX (4096). 8 KiB leaves room for both, while a server that
-      # never sends CRLF can't grow the read buffer without bound.
-      MAX_LINE_BYTES = 8 * 1024
-
       def read_line
-        flush_write_buffer
-        data = @sock.gets("\r\n", MAX_LINE_BYTES)
+        data = @sock.gets("\r\n")
         error_on_request!('EOF in read_line') if data.nil?
-        # Without its CRLF the line either ran past MAX_LINE_BYTES or was cut
-        # short by EOF; either way the connection is out of step
-        error_on_request!("Reply line without CRLF in the first #{MAX_LINE_BYTES} bytes") unless data.end_with?("\r\n")
         data
       rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
         error_on_request!(e)
       end
 
-      # memcached can't store an item larger than 1 GiB (its -I maximum), so a
-      # reply claiming more (or a negative size) is malformed or hostile.
-      # Reading allocates the full count up front, so check before reading.
-      MAX_READ_BYTES = (1024 * 1024 * 1024) + 2 # plus the value's trailing "\r\n"
-
-      def check_read_size!(count)
-        return if count.between?(0, MAX_READ_BYTES)
-
-        raise Dalli::DalliError, "Reply size #{count} from #{name} is out of range"
-      end
-
       def read(count)
-        check_read_size!(count)
-        flush_write_buffer
-        @sock.readfull(count)
+        # JRuby doesn't support IO#timeout=, so use custom readfull implementation
+        # CRuby 3.3+ has IO#timeout= which makes IO#read work with timeouts
+        if RUBY_ENGINE == 'jruby'
+          @sock.readfull(count)
+        else
+          @sock.read(count)
+        end
       rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
         error_on_request!(e)
       end
 
-      # Requests are buffered here rather than in the socket's own IO buffer
-      # (the socket is sync). Ruby flushes an IO's write buffer when any process
-      # holding it closes or finalizes the IO, so after a fork a child could
-      # send bytes the parent had buffered but not sent yet: a quiet write would
-      # run twice, or the parent's replies would shift onto the wrong requests.
-      # A forked child discards this buffer instead. Like IO's own buffer, it
-      # is sent once it grows past this size, or when a reply is read.
-      WRITE_BUFFER_FLUSH_BYTES = 64 * 1024
-
       def write(bytes)
-        # A copy: the caller may change its string before the buffer is sent
-        # (the meta protocol writes a value string as its own part), as Ruby's
-        # IO buffer copied on write
-        @write_buffer << (bytes.frozen? ? bytes : bytes.dup)
-        @write_buffer_bytes += bytes.bytesize
-        flush_write_buffer if @write_buffer_bytes >= WRITE_BUFFER_FLUSH_BYTES
-        bytes.bytesize
-      rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, IOError => e
+        @sock.write(bytes)
+      rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS => e
         error_on_request!(e)
       end
 
       def flush
-        flush_write_buffer
-      rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, IOError => e
+        @sock.flush
+      rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS => e
         error_on_request!(e)
       end
 
       # Non-blocking read.  Here to support the operation
       # of the get_multi operation
       def read_nonblock
-        flush_write_buffer
         @sock.read_available
       end
 
@@ -286,9 +208,8 @@ module Dalli
         raise Dalli::RetryableNetworkError, message
       end
 
-      # Called on connect. Deliberately leaves @fail_count alone; see
-      # finish_request!.
       def reset_down_info
+        @fail_count = 0
         @down_at = nil
         @last_down_at = nil
         @msg = nil
@@ -313,18 +234,13 @@ module Dalli
       def reconnect_on_fork
         message = 'Fork detected, re-connecting child process...'
         Dalli.logger.info { message }
-        # Drops anything the parent had buffered, closes the inherited socket
-        # without touching the parent's connection, and reconnects immediately
+        # Close socket on a fork and reconnect immediately
         close
         establish_connection
       end
 
       def fork_detected?
-        # Process.pid rather than PIDCache: PIDCache is refreshed by a
-        # Process._fork hook, and another library's fork hook (such as
-        # connection_pool closing its connections) can run in the child before
-        # it, which would close the parent's TLS session as if it were ours.
-        @pid && @pid != Process.pid
+        @pid && @pid != PIDCache.pid
       end
 
       def log_down_detected
@@ -345,39 +261,6 @@ module Dalli
         time = Time.now - @down_at
         Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
       end
-
-      private
-
-      # One write per flush. A TLS socket sends each argument of
-      # write(*parts) as its own record and system call, so the parts are
-      # joined first.
-      def flush_write_buffer
-        return if @write_buffer.empty?
-
-        @sock.write(@write_buffer.size == 1 ? @write_buffer.first : joined_write_buffer)
-        discard_write_buffer
-      end
-
-      # Joined as bytes: requests can mix UTF-8 and binary strings, which
-      # String#<< refuses to combine when both hold non-ASCII bytes
-      if String.method_defined?(:append_as_bytes)
-        def joined_write_buffer
-          out = String.new(capacity: @write_buffer_bytes, encoding: Encoding::BINARY)
-          @write_buffer.each { |part| out.append_as_bytes(part) }
-          out
-        end
-      else
-        def joined_write_buffer
-          out = String.new(capacity: @write_buffer_bytes, encoding: Encoding::BINARY)
-          @write_buffer.each { |part| out << part.b }
-          out
-        end
-      end
-
-      def discard_write_buffer
-        @write_buffer.clear
-        @write_buffer_bytes = 0
-      end
     end
   end
 end