dalli 3.0.6 → 3.1.3

data/lib/dalli/protocol/binary.rb

@@ -1,14 +1,9 @@
 # frozen_string_literal: true
 
-require 'English'
 require 'forwardable'
 require 'socket'
 require 'timeout'
 
-require_relative 'binary/request_formatter'
-require_relative 'binary/response_processor'
-require_relative 'binary/sasl_authentication'
-
 module Dalli
   module Protocol
     ##
@@ -16,383 +11,70 @@ module Dalli
     # protocol.  Contains logic for managing connection state to the server (retries, etc),
     # formatting requests to the server, and unpacking responses.
     ##
-    class Binary
-      extend Forwardable
-
-      attr_accessor :hostname, :port, :weight, :options
-      attr_reader :sock, :socket_type
-
-      def_delegators :@value_marshaller, :serializer, :compressor, :compression_min_size, :compress_by_default?
-
-      DEFAULTS = {
-        # seconds between trying to contact a remote server
-        down_retry_delay: 30,
-        # connect/read/write timeout for socket operations
-        socket_timeout: 1,
-        # times a socket operation may fail before considering the server dead
-        socket_max_failures: 2,
-        # amount of time to sleep between retries when a failure occurs
-        socket_failure_delay: 0.1,
-        username: nil,
-        password: nil
-      }.freeze
-
-      def initialize(attribs, options = {})
-        @hostname, @port, @weight, @socket_type, options = ServerConfigParser.parse(attribs, options)
-        @options = DEFAULTS.merge(options)
-        @value_marshaller = ValueMarshaller.new(@options)
-        @response_processor = ResponseProcessor.new(self, @value_marshaller)
-
-        reset_down_info
-        @sock = nil
-        @pid = nil
-        @request_in_progress = false
-      end
-
-      def name
-        if socket_type == :unix
-          hostname
-        else
-          "#{hostname}:#{port}"
-        end
-      end
-
-      # Chokepoint method for error handling and ensuring liveness
-      def request(opcode, *args)
-        verify_state
-        # The alive? call has the side effect of connecting the underlying
-        # socket if it is not connected, or there's been a disconnect
-        # because of timeout or other error.  Method raises an error
-        # if it can't connect
-        raise_memcached_down_err unless alive?
-
-        begin
-          send(opcode, *args)
-        rescue Dalli::MarshalError => e
-          log_marshall_err(args.first, e)
-          raise
-        rescue Dalli::DalliError, Dalli::NetworkError, Dalli::ValueOverMaxSize, Timeout::Error
-          raise
-        rescue StandardError => e
-          log_unexpected_err(e)
-          down!
-        end
-      end
-
-      def raise_memcached_down_err
-        raise Dalli::NetworkError,
-              "#{name} is down: #{@error} #{@msg}. If you are sure it is running, "\
-              "ensure memcached version is > #{::Dalli::MIN_SUPPORTED_MEMCACHED_VERSION}."
-      end
-
-      def log_marshall_err(key, err)
-        Dalli.logger.error "Marshalling error for key '#{key}': #{err.message}"
-        Dalli.logger.error 'You are trying to cache a Ruby object which cannot be serialized to memcached.'
-      end
-
-      def log_unexpected_err(err)
-        Dalli.logger.error "Unexpected exception during Dalli request: #{err.class.name}: #{err.message}"
-        Dalli.logger.error err.backtrace.join("\n\t")
-      end
-
-      # The socket connection to the underlying server is initialized as a side
-      # effect of this call.  In fact, this is the ONLY place where that
-      # socket connection is initialized.
-      def alive?
-        return true if @sock
-        return false unless reconnect_down_server?
-
-        connect
-        !!@sock
-      rescue Dalli::NetworkError
-        false
-      end
-
-      def reconnect_down_server?
-        return true unless @last_down_at
-
-        time_to_next_reconnect = @last_down_at + options[:down_retry_delay] - Time.now
-        return true unless time_to_next_reconnect.positive?
-
-        Dalli.logger.debug do
-          format('down_retry_delay not reached for %<name>s (%<time>.3f seconds left)', name: name,
-                                                                                        time: time_to_next_reconnect)
-        end
-        false
-      end
-
-      # Closes the underlying socket and cleans up
-      # socket state.
-      def close
-        return unless @sock
-
-        begin
-          @sock.close
-        rescue StandardError
-          nil
-        end
-        @sock = nil
-        @pid = nil
-        abort_request!
-      end
-
-      def lock!; end
-
-      def unlock!; end
-
-      # Start reading key/value pairs from this connection. This is usually called
-      # after a series of GETKQ commands. A NOOP is sent, and the server begins
-      # flushing responses for kv pairs that were found.
-      #
-      # Returns nothing.
-      def multi_response_start
-        verify_state
-        write_noop
-        @multi_buffer = +''
-        @position = 0
-        start_request!
-      end
-
-      # Did the last call to #multi_response_start complete successfully?
-      def multi_response_completed?
-        @multi_buffer.nil?
-      end
-
-      # Attempt to receive and parse as many key/value pairs as possible
-      # from this server. After #multi_response_start, this should be invoked
-      # repeatedly whenever this server's socket is readable until
-      # #multi_response_completed?.
-      #
-      # Returns a Hash of kv pairs received.
-      def multi_response_nonblock
-        reconnect! 'multi_response has completed' if @multi_buffer.nil?
-
-        @multi_buffer << @sock.read_available
-        buf = @multi_buffer
-        pos = @position
-        values = {}
-
-        while buf.bytesize - pos >= ResponseProcessor::RESP_HEADER_SIZE
-          header = buf.slice(pos, ResponseProcessor::RESP_HEADER_SIZE)
-          _, extra_len, key_len, body_len, cas = @response_processor.unpack_header(header)
-
-          # We've reached the noop at the end of the pipeline
-          if key_len.zero?
-            finish_multi_response
-            break
-          end
-
-          # Break and read more unless we already have the entire response for this header
-          resp_size = ResponseProcessor::RESP_HEADER_SIZE + body_len
-          break unless buf.bytesize - pos >= resp_size
-
-          body = buf.slice(pos + ResponseProcessor::RESP_HEADER_SIZE, body_len)
-          begin
-            key, value = @response_processor.unpack_response_body(extra_len, key_len, body, true)
-            values[key] = [value, cas]
-          rescue DalliError
-            # TODO: Determine if we should be swallowing
-            # this error
-          end
-
-          pos = pos + ResponseProcessor::RESP_HEADER_SIZE + body_len
-        end
-        # TODO: We should be discarding the already processed buffer at this point
-        @position = pos
-
-        values
-      rescue SystemCallError, Timeout::Error, EOFError => e
-        failure!(e)
-      end
-
-      def finish_multi_response
-        @multi_buffer = nil
-        @position = nil
-        finish_request!
-      end
-
-      # Abort an earlier #multi_response_start. Used to signal an external
-      # timeout. The underlying socket is disconnected, and the exception is
-      # swallowed.
-      #
-      # Returns nothing.
-      def multi_response_abort
-        @multi_buffer = nil
-        @position = nil
-        abort_request!
-        return true unless @sock
-
-        failure!(RuntimeError.new('External timeout'))
-      rescue NetworkError
-        true
-      end
-
-      def read(count)
-        start_request!
-        data = @sock.readfull(count)
-        finish_request!
-        data
-      rescue SystemCallError, Timeout::Error, EOFError => e
-        failure!(e)
-      end
-
-      def write(bytes)
-        start_request!
-        result = @sock.write(bytes)
-        finish_request!
-        result
-      rescue SystemCallError, Timeout::Error => e
-        failure!(e)
-      end
-
-      def socket_timeout
-        @socket_timeout ||= @options[:socket_timeout]
+    class Binary < Base
+      def response_processor
+        @response_processor ||= ResponseProcessor.new(@connection_manager, @value_marshaller)
       end
 
-      # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
-
       private
 
-      def request_in_progress?
-        @request_in_progress
-      end
-
-      def start_request!
-        @request_in_progress = true
-      end
-
-      def finish_request!
-        @request_in_progress = false
-      end
-
-      def abort_request!
-        @request_in_progress = false
-      end
-
-      def verify_state
-        failure!(RuntimeError.new('Already writing to socket')) if request_in_progress?
-        reconnect_on_fork if fork_detected?
-      end
-
-      def fork_detected?
-        @pid && @pid != Process.pid
-      end
-
-      def reconnect_on_fork
-        message = 'Fork detected, re-connecting child process...'
-        Dalli.logger.info { message }
-        reconnect! message
-      end
-
-      # Marks the server instance as needing reconnect.  Raises a
-      # Dalli::NetworkError with the specified message.  Calls close
-      # to clean up socket state
-      def reconnect!(message)
-        close
-        sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
-        raise Dalli::NetworkError, message
-      end
-
-      # Raises Dalli::NetworkError
-      def failure!(exception)
-        message = "#{name} failed (count: #{@fail_count}) #{exception.class}: #{exception.message}"
-        Dalli.logger.warn { message }
-
-        @fail_count += 1
-        if @fail_count >= options[:socket_max_failures]
-          down!
-        else
-          reconnect! 'Socket operation failed, retrying...'
-        end
-      end
-
-      # Marks the server instance as down.  Updates the down_at state
-      # and raises an Dalli::NetworkError that includes the underlying
-      # error in the message.  Calls close to clean up socket state
-      def down!
-        close
-        log_down_detected
-
-        @error = $ERROR_INFO&.class&.name
-        @msg ||= $ERROR_INFO&.message
-        raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
-      end
-
-      def log_down_detected
-        @last_down_at = Time.now
-
-        if @down_at
-          time = Time.now - @down_at
-          Dalli.logger.debug { format('%<name>s is still down (for %<time>.3f seconds now)', name: name, time: time) }
-        else
-          @down_at = @last_down_at
-          Dalli.logger.warn("#{name} is down")
-        end
-      end
-
-      def log_up_detected
-        return unless @down_at
+      ALLOWED_QUIET_OPS = %i[add replace set delete incr decr append prepend flush noop].freeze
+      def verify_allowed_quiet!(opkey)
+        return if ALLOWED_QUIET_OPS.include?(opkey)
 
-        time = Time.now - @down_at
-        Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
+        raise Dalli::NotPermittedMultiOpError, "The operation #{opkey} is not allowed in a quiet block."
       end
 
-      def up!
-        log_up_detected
-        reset_down_info
+      # Retrieval Commands
+      def get(key, options = nil)
+        req = RequestFormatter.standard_request(opkey: :get, key: key)
+        write(req)
+        response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
       end
 
-      def reset_down_info
-        @fail_count = 0
-        @down_at = nil
-        @last_down_at = nil
-        @msg = nil
-        @error = nil
+      def quiet_get_request(key)
+        RequestFormatter.standard_request(opkey: :getkq, key: key)
       end
 
-      def multi?
-        Thread.current[:dalli_multi]
+      def gat(key, ttl, options = nil)
+        ttl = TtlSanitizer.sanitize(ttl)
+        req = RequestFormatter.standard_request(opkey: :gat, key: key, ttl: ttl)
+        write(req)
+        response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
       end
 
-      def cache_nils?(opts)
-        return false unless opts.is_a?(Hash)
-
-        opts[:cache_nils] ? true : false
+      def touch(key, ttl)
+        ttl = TtlSanitizer.sanitize(ttl)
+        write(RequestFormatter.standard_request(opkey: :touch, key: key, ttl: ttl))
+        response_processor.generic_response
       end
 
-      def get(key, options = nil)
+      # TODO: This is confusing, as there's a cas command in memcached
+      # and this isn't it.  Maybe rename?  Maybe eliminate?
+      def cas(key)
         req = RequestFormatter.standard_request(opkey: :get, key: key)
         write(req)
-        @response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
-      end
-
-      def send_multiget(keys)
-        req = +''
-        keys.each do |key|
-          req << RequestFormatter.standard_request(opkey: :getkq, key: key)
-        end
-        # Could send noop here instead of in multi_response_start
-        write(req)
+        response_processor.data_cas_response
       end
 
+      # Storage Commands
       def set(key, value, ttl, cas, options)
-        opkey = multi? ? :setq : :set
-        process_value_req(opkey, key, value, ttl, cas, options)
+        opkey = quiet? ? :setq : :set
+        storage_req(opkey, key, value, ttl, cas, options)
       end
 
       def add(key, value, ttl, options)
-        opkey = multi? ? :addq : :add
-        cas = 0
-        process_value_req(opkey, key, value, ttl, cas, options)
+        opkey = quiet? ? :addq : :add
+        storage_req(opkey, key, value, ttl, 0, options)
       end
 
       def replace(key, value, ttl, cas, options)
-        opkey = multi? ? :replaceq : :replace
-        process_value_req(opkey, key, value, ttl, cas, options)
+        opkey = quiet? ? :replaceq : :replace
+        storage_req(opkey, key, value, ttl, cas, options)
       end
 
       # rubocop:disable Metrics/ParameterLists
-      def process_value_req(opkey, key, value, ttl, cas, options)
+      def storage_req(opkey, key, value, ttl, cas, options)
         (value, bitflags) = @value_marshaller.store(key, value, options)
         ttl = TtlSanitizer.sanitize(ttl)
 
@@ -400,21 +82,42 @@ module Dalli
                                                 value: value, bitflags: bitflags,
                                                 ttl: ttl, cas: cas)
         write(req)
-        @response_processor.cas_response unless multi?
+        response_processor.storage_response unless quiet?
       end
       # rubocop:enable Metrics/ParameterLists
 
+      def append(key, value)
+        opkey = quiet? ? :appendq : :append
+        write_append_prepend opkey, key, value
+      end
+
+      def prepend(key, value)
+        opkey = quiet? ? :prependq : :prepend
+        write_append_prepend opkey, key, value
+      end
+
+      def write_append_prepend(opkey, key, value)
+        write(RequestFormatter.standard_request(opkey: opkey, key: key, value: value))
+        response_processor.no_body_response unless quiet?
+      end
+
+      # Delete Commands
       def delete(key, cas)
-        opkey = multi? ? :deleteq : :delete
+        opkey = quiet? ? :deleteq : :delete
         req = RequestFormatter.standard_request(opkey: opkey, key: key, cas: cas)
         write(req)
-        @response_processor.generic_response unless multi?
+        response_processor.delete_response unless quiet?
       end
 
-      def flush(ttl = 0)
-        req = RequestFormatter.standard_request(opkey: :flush, ttl: ttl)
-        write(req)
-        @response_processor.generic_response
+      # Arithmetic Commands
+      def decr(key, count, ttl, initial)
+        opkey = quiet? ? :decrq : :decr
+        decr_incr opkey, key, count, ttl, initial
+      end
+
+      def incr(key, count, ttl, initial)
+        opkey = quiet? ? :incrq : :incr
+        decr_incr opkey, key, count, ttl, initial
       end
 
       # This allows us to special case a nil initial value, and
@@ -429,115 +132,48 @@ module Dalli
         initial ||= 0
         write(RequestFormatter.decr_incr_request(opkey: opkey, key: key,
                                                  count: count, initial: initial, expiry: expiry))
-        @response_processor.decr_incr_response
-      end
-
-      def decr(key, count, ttl, initial)
-        decr_incr :decr, key, count, ttl, initial
-      end
-
-      def incr(key, count, ttl, initial)
-        decr_incr :incr, key, count, ttl, initial
-      end
-
-      def write_append_prepend(opkey, key, value)
-        write_generic RequestFormatter.standard_request(opkey: opkey, key: key, value: value)
+        response_processor.decr_incr_response unless quiet?
       end
 
-      def write_generic(bytes)
-        write(bytes)
-        @response_processor.generic_response
-      end
-
-      def write_noop
-        req = RequestFormatter.standard_request(opkey: :noop)
-        write(req)
+      # Other Commands
+      def flush(ttl = 0)
+        opkey = quiet? ? :flushq : :flush
+        write(RequestFormatter.standard_request(opkey: opkey, ttl: ttl))
+        response_processor.no_body_response unless quiet?
       end
 
       # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
       # We need to read all the responses at once.
       def noop
         write_noop
-        @response_processor.multi_with_keys_response
-      end
-
-      def append(key, value)
-        write_append_prepend :append, key, value
-      end
-
-      def prepend(key, value)
-        write_append_prepend :prepend, key, value
+        response_processor.multi_with_keys_response
       end
 
       def stats(info = '')
         req = RequestFormatter.standard_request(opkey: :stat, key: info)
         write(req)
-        @response_processor.multi_with_keys_response
+        response_processor.multi_with_keys_response
       end
 
       def reset_stats
-        write_generic RequestFormatter.standard_request(opkey: :stat, key: 'reset')
-      end
-
-      def cas(key)
-        req = RequestFormatter.standard_request(opkey: :get, key: key)
-        write(req)
-        @response_processor.data_cas_response
+        write(RequestFormatter.standard_request(opkey: :stat, key: 'reset'))
+        response_processor.generic_response
       end
 
       def version
-        write_generic RequestFormatter.standard_request(opkey: :version)
+        write(RequestFormatter.standard_request(opkey: :version))
+        response_processor.generic_response
       end
 
-      def touch(key, ttl)
-        ttl = TtlSanitizer.sanitize(ttl)
-        write_generic RequestFormatter.standard_request(opkey: :touch, key: key, ttl: ttl)
-      end
-
-      def gat(key, ttl, options = nil)
-        ttl = TtlSanitizer.sanitize(ttl)
-        req = RequestFormatter.standard_request(opkey: :gat, key: key, ttl: ttl)
+      def write_noop
+        req = RequestFormatter.standard_request(opkey: :noop)
         write(req)
-        @response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
-      end
-
-      def connect
-        Dalli.logger.debug { "Dalli::Server#connect #{name}" }
-
-        begin
-          @pid = Process.pid
-          @sock = memcached_socket
-          authenticate_connection if require_auth?
-          @version = version # Connect socket if not authed
-          up!
-        rescue Dalli::DalliError # SASL auth failure
-          raise
-        rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
-          # SocketError = DNS resolution failure
-          failure!(e)
-        end
-      end
-
-      def memcached_socket
-        if socket_type == :unix
-          Dalli::Socket::UNIX.open(hostname, self, options)
-        else
-          Dalli::Socket::TCP.open(hostname, port, self, options)
-        end
-      end
-
-      def require_auth?
-        !username.nil?
-      end
-
-      def username
-        @options[:username] || ENV['MEMCACHE_USERNAME']
-      end
-
-      def password
-        @options[:password] || ENV['MEMCACHE_PASSWORD']
       end
 
+      require_relative 'binary/request_formatter'
+      require_relative 'binary/response_header'
+      require_relative 'binary/response_processor'
+      require_relative 'binary/sasl_authentication'
       include SaslAuthentication
     end
   end