dalli 3.0.6 → 3.1.3

data/lib/dalli/pipelined_getter.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module Dalli
+  ##
+  # Contains logic for the pipelined gets implemented by the client.
+  ##
+  class PipelinedGetter
+    def initialize(ring, key_manager)
+      @ring = ring
+      @key_manager = key_manager
+    end
+
+    ##
+    # Yields, one at a time, keys and their values+attributes.
+    #
+    def process(keys, &block)
+      return {} if keys.empty?
+
+      @ring.lock do
+        servers = setup_requests(keys)
+        start_time = Time.now
+        servers = fetch_responses(servers, start_time, @ring.socket_timeout, &block) until servers.empty?
+      end
+    rescue NetworkError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { 'retrying pipelined gets because of timeout' }
+      retry
+    end
+
+    def setup_requests(keys)
+      groups = groups_for_keys(keys)
+      make_getkq_requests(groups)
+
+      # TODO: How does this exit on a NetworkError
+      finish_queries(groups.keys)
+    end
+
+    ##
+    # Loop through the server-grouped sets of keys, writing
+    # the corresponding getkq requests to the appropriate servers
+    #
+    # It's worth noting that we could potentially reduce bytes
+    # on the wire by switching from getkq to getq, and using
+    # the opaque value to match requests to responses.
+    ##
+    def make_getkq_requests(groups)
+      groups.each do |server, keys_for_server|
+        server.request(:pipelined_get, keys_for_server)
+      rescue DalliError, NetworkError => e
+        Dalli.logger.debug { e.inspect }
+        Dalli.logger.debug { "unable to get keys for server #{server.name}" }
+      end
+    end
+
+    ##
+    # This loops through the servers that have keys in
+    # our set, sending the noop to terminate the set of queries.
+    ##
+    def finish_queries(servers)
+      deleted = []
+
+      servers.each do |server|
+        next unless server.alive?
+
+        begin
+          finish_query_for_server(server)
+        rescue Dalli::NetworkError
+          raise
+        rescue Dalli::DalliError
+          deleted.append(server)
+        end
+      end
+
+      servers.delete_if { |server| deleted.include?(server) }
+    rescue Dalli::NetworkError
+      abort_without_timeout(servers)
+      raise
+    end
+
+    def finish_query_for_server(server)
+      server.pipeline_response_setup
+    rescue Dalli::NetworkError
+      raise
+    rescue Dalli::DalliError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { "Results from server: #{server.name} will be missing from the results" }
+      raise
+    end
+
+    # Swallows Dalli::NetworkError
+    def abort_without_timeout(servers)
+      servers.each(&:pipeline_abort)
+    end
+
+    def fetch_responses(servers, start_time, timeout, &block)
+      # Remove any servers which are not connected
+      servers.delete_if { |s| !s.connected? }
+      return [] if servers.empty?
+
+      time_left = remaining_time(start_time, timeout)
+      readable_servers = servers_with_response(servers, time_left)
+      if readable_servers.empty?
+        abort_with_timeout(servers)
+        return []
+      end
+
+      # Loop through the servers with responses, and
+      # delete any from our list that are finished
+      readable_servers.each do |server|
+        servers.delete(server) if process_server(server, &block)
+      end
+      servers
+    rescue NetworkError
+      # Abort and raise if we encountered a network error.  This triggers
+      # a retry at the top level.
+      abort_without_timeout(servers)
+      raise
+    end
+
+    def remaining_time(start, timeout)
+      elapsed = Time.now - start
+      return 0 if elapsed > timeout
+
+      timeout - elapsed
+    end
+
+    # Swallows Dalli::NetworkError
+    def abort_with_timeout(servers)
+      abort_without_timeout(servers)
+      servers.each do |server|
+        Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
+      end
+
+      true # Required to simplify caller
+    end
+
+    # Processes responses from a server.  Returns true if there are no
+    # additional responses from this server.
+    def process_server(server)
+      server.pipeline_next_responses.each_pair do |key, value_list|
+        yield @key_manager.key_without_namespace(key), value_list
+      end
+
+      server.pipeline_complete?
+    end
+
+    def servers_with_response(servers, timeout)
+      return [] if servers.empty?
+
+      # TODO: - This is a bit challenging.  Essentially the PipelinedGetter
+      # is a reactor, but without the benefit of a Fiber or separate thread.
+      # My suspicion is that we may want to try and push this down into the
+      # individual servers, but I'm not sure.  For now, we keep the
+      # mapping between the alerted object (the socket) and the
+      # corrresponding server here.
+      server_map = servers.each_with_object({}) { |s, h| h[s.sock] = s }
+
+      readable, = IO.select(server_map.keys, nil, nil, timeout)
+      return [] if readable.nil?
+
+      readable.map { |sock| server_map[sock] }
+    end
+
+    def groups_for_keys(*keys)
+      keys.flatten!
+      keys.map! { |a| @key_manager.validate_key(a.to_s) }
+      groups = @ring.keys_grouped_by_server(keys)
+      if (unfound_keys = groups.delete(nil))
+        Dalli.logger.debug do
+          "unable to get keys for #{unfound_keys.length} keys "\
+            'because no matching server was found'
+        end
+      end
+      groups
+    end
+  end
+end

data/lib/dalli/protocol/base.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'socket'
+require 'timeout'
+
+module Dalli
+  module Protocol
+    ##
+    # Base class for a single Memcached server, containing logic common to all
+    # protocols.  Contains logic for managing connection state to the server and value
+    # handling.
+    ##
+    class Base
+      extend Forwardable
+
+      attr_accessor :weight, :options
+
+      def_delegators :@value_marshaller, :serializer, :compressor, :compression_min_size, :compress_by_default?
+      def_delegators :@connection_manager, :name, :sock, :hostname, :port, :close, :connected?, :socket_timeout,
+                     :socket_type, :up!, :down!, :write, :reconnect_down_server?, :raise_down_error
+
+      def initialize(attribs, client_options = {})
+        hostname, port, socket_type, @weight, user_creds = ServerConfigParser.parse(attribs)
+        @options = client_options.merge(user_creds)
+        @value_marshaller = ValueMarshaller.new(@options)
+        @connection_manager = ConnectionManager.new(hostname, port, socket_type, @options)
+      end
+
+      # Chokepoint method for error handling and ensuring liveness
+      def request(opkey, *args)
+        verify_state(opkey)
+
+        begin
+          send(opkey, *args)
+        rescue Dalli::MarshalError => e
+          log_marshal_err(args.first, e)
+          raise
+        rescue Dalli::DalliError
+          raise
+        rescue StandardError => e
+          log_unexpected_err(e)
+          down!
+        end
+      end
+
+      ##
+      # Boolean method used by clients of this class to determine if this
+      # particular memcached instance is available for use.
+      def alive?
+        ensure_connected!
+      rescue Dalli::NetworkError
+        # ensure_connected! raises a NetworkError if connection fails.  We
+        # want to capture that error and convert it to a boolean value here.
+        false
+      end
+
+      def lock!; end
+
+      def unlock!; end
+
+      # Start reading key/value pairs from this connection. This is usually called
+      # after a series of GETKQ commands. A NOOP is sent, and the server begins
+      # flushing responses for kv pairs that were found.
+      #
+      # Returns nothing.
+      def pipeline_response_setup
+        verify_state(:getkq)
+        write_noop
+        response_buffer.reset
+        @connection_manager.start_request!
+      end
+
+      # Attempt to receive and parse as many key/value pairs as possible
+      # from this server. After #pipeline_response_setup, this should be invoked
+      # repeatedly whenever this server's socket is readable until
+      # #pipeline_complete?.
+      #
+      # Returns a Hash of kv pairs received.
+      def pipeline_next_responses
+        reconnect_on_pipeline_complete!
+        values = {}
+
+        response_buffer.read
+
+        resp_header, key, value = pipeline_response
+        # resp_header is not nil only if we have a full response to parse
+        # in the buffer
+        while resp_header
+          # If the status is ok and key is nil, then this is the response
+          # to the noop at the end of the pipeline
+          finish_pipeline && break if resp_header.ok? && key.nil?
+
+          # If the status is ok and the key is not nil, then this is a
+          # getkq response with a value that we want to set in the response hash
+          values[key] = [value, resp_header.cas] unless key.nil?
+
+          # Get the next response from the buffer
+          resp_header, key, value = pipeline_response
+        end
+
+        values
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        @connection_manager.error_on_request!(e)
+      end
+
+      # Abort current pipelined get. Generally used to signal an external
+      # timeout during pipelined get.  The underlying socket is
+      # disconnected, and the exception is swallowed.
+      #
+      # Returns nothing.
+      def pipeline_abort
+        response_buffer.clear
+        @connection_manager.abort_request!
+        return true unless connected?
+
+        # Closes the connection, which ensures that our connection
+        # is in a clean state for future requests
+        @connection_manager.error_on_request!('External timeout')
+      rescue NetworkError
+        true
+      end
+
+      # Did the last call to #pipeline_response_setup complete successfully?
+      def pipeline_complete?
+        !response_buffer.in_progress?
+      end
+
+      def username
+        @options[:username] || ENV['MEMCACHE_USERNAME']
+      end
+
+      def password
+        @options[:password] || ENV['MEMCACHE_PASSWORD']
+      end
+
+      def require_auth?
+        !username.nil?
+      end
+
+      def quiet?
+        Thread.current[::Dalli::QUIET]
+      end
+      alias multi? quiet?
+
+      # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
+
+      private
+
+      ##
+      # Checks to see if we can execute the specified operation.  Checks
+      # whether the connection is in use, and whether the command is allowed
+      ##
+      def verify_state(opkey)
+        @connection_manager.confirm_ready!
+        verify_allowed_quiet!(opkey) if quiet?
+
+        # The ensure_connected call has the side effect of connecting the
+        # underlying socket if it is not connected, or there's been a disconnect
+        # because of timeout or other error.  Method raises an error
+        # if it can't connect
+        raise_down_error unless ensure_connected!
+      end
+
+      # The socket connection to the underlying server is initialized as a side
+      # effect of this call.  In fact, this is the ONLY place where that
+      # socket connection is initialized.
+      #
+      # Both this method and connect need to be in this class so we can do auth
+      # as required
+      #
+      # Since this is invoked exclusively in verify_state!, we don't need to worry about
+      # thread safety.  Using it elsewhere may require revisiting that assumption.
+      def ensure_connected!
+        return true if connected?
+        return false unless reconnect_down_server?
+
+        connect # This call needs to be in this class so we can do auth
+        connected?
+      end
+
+      def cache_nils?(opts)
+        return false unless opts.is_a?(Hash)
+
+        opts[:cache_nils] ? true : false
+      end
+
+      def connect
+        @connection_manager.establish_connection
+        authenticate_connection if require_auth?
+        @version = version # Connect socket if not authed
+        up!
+      rescue Dalli::DalliError
+        raise
+      end
+
+      def pipelined_get(keys)
+        req = +''
+        keys.each do |key|
+          req << quiet_get_request(key)
+        end
+        # Could send noop here instead of in pipeline_response_setup
+        write(req)
+      end
+
+      def response_buffer
+        @response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
+      end
+
+      def pipeline_response
+        response_buffer.process_single_getk_response
+      end
+
+      # Called after the noop response is received at the end of a set
+      # of pipelined gets
+      def finish_pipeline
+        response_buffer.clear
+        @connection_manager.finish_request!
+
+        true # to simplify response
+      end
+
+      def reconnect_on_pipeline_complete!
+        @connection_manager.reconnect! 'pipelined get has completed' if pipeline_complete?
+      end
+
+      def log_marshal_err(key, err)
+        Dalli.logger.error "Marshalling error for key '#{key}': #{err.message}"
+        Dalli.logger.error 'You are trying to cache a Ruby object which cannot be serialized to memcached.'
+      end
+
+      def log_unexpected_err(err)
+        Dalli.logger.error "Unexpected exception during Dalli request: #{err.class.name}: #{err.message}"
+        Dalli.logger.error err.backtrace.join("\n\t")
+      end
+    end
+  end
+end

data/lib/dalli/protocol/binary/request_formatter.rb

@@ -31,11 +31,14 @@ module Dalli
           deleteq: 0x14,
           incrq: 0x15,
           decrq: 0x16,
+          flushq: 0x18,
+          appendq: 0x19,
+          prependq: 0x1A,
+          touch: 0x1C,
+          gat: 0x1D,
           auth_negotiation: 0x20,
           auth_request: 0x21,
-          auth_continue: 0x22,
-          touch: 0x1C,
-          gat: 0x1D
+          auth_continue: 0x22
         }.freeze
 
         REQ_HEADER_FORMAT = 'CCnCCnNNQ'
@@ -56,6 +59,8 @@ module Dalli
 
           append: KEY_AND_VALUE,
           prepend: KEY_AND_VALUE,
+          appendq: KEY_AND_VALUE,
+          prependq: KEY_AND_VALUE,
           auth_request: KEY_AND_VALUE,
           auth_continue: KEY_AND_VALUE,
 
@@ -68,8 +73,11 @@ module Dalli
 
           incr: INCR_DECR,
           decr: INCR_DECR,
+          incrq: INCR_DECR,
+          decrq: INCR_DECR,
 
           flush: TTL_ONLY,
+          flushq: TTL_ONLY,
 
           noop: NO_BODY,
           auth_negotiation: NO_BODY,

data/lib/dalli/protocol/binary/response_header.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Binary
+      ##
+      # Class that encapsulates data parsed from a memcached response header.
+      ##
+      class ResponseHeader
+        SIZE = 24
+        FMT = '@2nCCnNNQ'
+
+        attr_reader :key_len, :extra_len, :data_type, :status, :body_len, :opaque, :cas
+
+        def initialize(buf)
+          raise ArgumentError, "Response buffer must be at least #{SIZE} bytes" unless buf.bytesize >= SIZE
+
+          @key_len, @extra_len, @data_type, @status, @body_len, @opaque, @cas = buf.unpack(FMT)
+        end
+
+        def ok?
+          status.zero?
+        end
+
+        def not_found?
+          status == 1
+        end
+
+        NOT_STORED_STATUSES = [2, 5].freeze
+        def not_stored?
+          NOT_STORED_STATUSES.include?(status)
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/binary/response_processor.rb

@@ -9,9 +9,6 @@ module Dalli
       # and parsing into local values.  Handles errors on unexpected values.
       ##
       class ResponseProcessor
-        RESP_HEADER = '@2nCCnNNQ'
-        RESP_HEADER_SIZE = 24
-
         # Response codes taken from:
         # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
         RESPONSE_CODES = {
@@ -44,14 +41,9 @@ module Dalli
         end
 
         def read_response
-          status, extra_len, key_len, body_len, cas = unpack_header(read_header)
-          body = read(body_len) if body_len.positive?
-          [status, extra_len, body, cas, key_len]
-        end
-
-        def unpack_header(header)
-          (key_len, extra_len, _, status, body_len, _, cas) = header.unpack(RESP_HEADER)
-          [status, extra_len, key_len, body_len, cas]
+          resp_header = ResponseHeader.new(read_header)
+          body = read(resp_header.body_len) if resp_header.body_len.positive?
+          [resp_header, body]
         end
 
         def unpack_response_body(extra_len, key_len, body, unpack)
@@ -63,45 +55,65 @@ module Dalli
         end
 
         def read_header
-          read(RESP_HEADER_SIZE) || raise(Dalli::NetworkError, 'No response')
+          read(ResponseHeader::SIZE) || raise(Dalli::NetworkError, 'No response')
         end
 
-        def not_found?(status)
-          status == 1
+        def raise_on_not_ok!(resp_header)
+          return if resp_header.ok?
+
+          raise Dalli::DalliError, "Response error #{resp_header.status}: #{RESPONSE_CODES[resp_header.status]}"
         end
 
-        NOT_STORED_STATUSES = [2, 5].freeze
-        def not_stored?(status)
-          NOT_STORED_STATUSES.include?(status)
+        def generic_response(unpack: false, cache_nils: false)
+          resp_header, body = read_response
+
+          return false if resp_header.not_stored? # Not stored, normal status for add operation
+          return cache_nils ? ::Dalli::NOT_FOUND : nil if resp_header.not_found?
+
+          raise_on_not_ok!(resp_header)
+          return true unless body
+
+          unpack_response_body(resp_header.extra_len, resp_header.key_len, body, unpack).last
         end
 
-        def raise_on_not_ok_status!(status)
-          return if status.zero?
+        ##
+        # Response for a storage operation.  Returns the cas on success.  False
+        # if the value wasn't stored.  And raises an error on all other error
+        # codes from memcached.
+        ##
+        def storage_response
+          resp_header, = read_response
+          return false if resp_header.not_stored? # Not stored, normal status for add operation
 
-          raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
+          raise_on_not_ok!(resp_header)
+          resp_header.cas
         end
 
-        def generic_response(unpack: false, cache_nils: false)
-          status, extra_len, body, _, key_len = read_response
+        def delete_response
+          resp_header, = read_response
+          return false if resp_header.not_found? || resp_header.not_stored?
 
-          return cache_nils ? ::Dalli::NOT_FOUND : nil if not_found?(status)
-          return false if not_stored?(status) # Not stored, normal status for add operation
+          raise_on_not_ok!(resp_header)
+          true
+        end
 
-          raise_on_not_ok_status!(status)
-          return true unless body
+        def no_body_response
+          resp_header, = read_response
+          return false if resp_header.not_stored? # Not stored, possible status for append/prepend
 
-          unpack_response_body(extra_len, key_len, body, unpack).last
+          raise_on_not_ok!(resp_header)
+          true
         end
 
-        def data_cas_response
-          status, extra_len, body, cas, key_len = read_response
-          return [nil, cas] if not_found?(status)
-          return [nil, false] if not_stored?(status)
+        def data_cas_response(unpack: true)
+          resp_header, body = read_response
+          return [nil, resp_header.cas] if resp_header.not_found?
+          return [nil, false] if resp_header.not_stored?
 
-          raise_on_not_ok_status!(status)
-          return [nil, cas] unless body
+          raise_on_not_ok!(resp_header)
+          return [nil, resp_header.cas] unless body
 
-          [unpack_response_body(extra_len, key_len, body, true).last, cas]
+          [unpack_response_body(resp_header.extra_len, resp_header.key_len, body, unpack).last, resp_header.cas]
         end
 
         def cas_response
@@ -111,17 +123,17 @@ module Dalli
         def multi_with_keys_response
           hash = {}
           loop do
-            status, extra_len, body, _, key_len = read_response
+            resp_header, body = read_response
             # This is the response to the terminating noop / end of stat
-            return hash if status.zero? && key_len.zero?
+            return hash if resp_header.ok? && resp_header.key_len.zero?
 
             # Ignore any responses with non-zero status codes,
             # such as errors from set operations.  That allows
             # this code to be used at the end of a multi
             # block to clear any error responses from inside the multi.
-            next unless status.zero?
+            next unless resp_header.ok?
 
-            key, value = unpack_response_body(extra_len, key_len, body, true)
+            key, value = unpack_response_body(resp_header.extra_len, resp_header.key_len, body, true)
             hash[key] = value
           end
         end
@@ -137,11 +149,58 @@ module Dalli
           raise Dalli::NetworkError, "Unexpected message format: #{extra_len} #{count}"
         end
 
-        def auth_response
-          (_, extra_len, _, status, body_len,) = read_header.unpack(RESP_HEADER)
-          validate_auth_format(extra_len, body_len)
+        def auth_response(buf = read_header)
+          resp_header = ResponseHeader.new(buf)
+          body_len = resp_header.body_len
+          validate_auth_format(resp_header.extra_len, body_len)
           content = read(body_len) if body_len.positive?
-          [status, content]
+          [resp_header.status, content]
+        end
+
+        def contains_header?(buf)
+          return false unless buf
+
+          buf.bytesize >= ResponseHeader::SIZE
+        end
+
+        def response_header_from_buffer(buf)
+          header = buf.slice(0, ResponseHeader::SIZE)
+          ResponseHeader.new(header)
+        end
+
+        ##
+        # This method returns an array of values used in a pipelined
+        # getk process.  The first value is the number of bytes by
+        # which to advance the pointer in the buffer.  If the
+        # complete response is found in the buffer, this will
+        # be the response size.  Otherwise it is zero.
+        #
+        # The remaining three values in the array are the ResponseHeader,
+        # key, and value.
+        ##
+        def getk_response_from_buffer(buf)
+          # There's no header in the buffer, so don't advance
+          return [0, nil, nil, nil] unless contains_header?(buf)
+
+          resp_header = response_header_from_buffer(buf)
+          body_len = resp_header.body_len
+
+          # We have a complete response that has no body.
+          # This is either the response to the terminating
+          # noop or, if the status is not zero, an intermediate
+          # error response that needs to be discarded.
+          return [ResponseHeader::SIZE, resp_header, nil, nil] if body_len.zero?
+
+          resp_size = ResponseHeader::SIZE + body_len
+          # The header is in the buffer, but the body is not.  As we don't have
+          # a complete response, don't advance the buffer
+          return [0, nil, nil, nil] unless buf.bytesize >= resp_size
+
+          # The full response is in our buffer, so parse it and return
+          # the values
+          body = buf.slice(ResponseHeader::SIZE, body_len)
+          key, value = unpack_response_body(resp_header.extra_len, resp_header.key_len, body, true)
+          [resp_size, resp_header, key, value]
         end
       end
     end

data/lib/dalli/protocol/binary/sasl_authentication.rb

@@ -10,7 +10,7 @@ module Dalli
         def perform_auth_negotiation
           write(RequestFormatter.standard_request(opkey: :auth_negotiation))
 
-          status, content = @response_processor.auth_response
+          status, content = response_processor.auth_response
           return [status, []] if content.nil?
 
           # Substitute spaces for the \x00 returned by