dalli 3.0.4 → 3.1.0

data/lib/dalli/protocol/binary.rb

@@ -1,17 +1,29 @@
 # frozen_string_literal: true
 
-require "socket"
-require "timeout"
+require 'English'
+require 'forwardable'
+require 'socket'
+require 'timeout'
+
+require_relative 'binary/request_formatter'
+require_relative 'binary/response_header'
+require_relative 'binary/response_processor'
+require_relative 'binary/sasl_authentication'
 
 module Dalli
   module Protocol
+    ##
+    # Access point for a single Memcached server, accessed via Memcached's binary
+    # protocol.  Contains logic for managing connection state to the server (retries, etc),
+    # formatting requests to the server, and unpacking responses.
+    ##
     class Binary
-      attr_accessor :hostname
-      attr_accessor :port
-      attr_accessor :weight
-      attr_accessor :options
-      attr_reader :sock
-      attr_reader :socket_type # possible values: :unix, :tcp
+      extend Forwardable
+
+      attr_accessor :hostname, :port, :weight, :options
+      attr_reader :sock, :socket_type
+
+      def_delegators :@value_marshaller, :serializer, :compressor, :compression_min_size, :compress_by_default?
 
       DEFAULTS = {
         # seconds between trying to contact a remote server
@@ -21,31 +33,24 @@ module Dalli
         # times a socket operation may fail before considering the server dead
         socket_max_failures: 2,
         # amount of time to sleep between retries when a failure occurs
-        socket_failure_delay: 0.1,
-        # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
-        value_max_bytes: 1024 * 1024,
-        serializer: Marshal,
-        username: nil,
-        password: nil,
-        keepalive: true,
-        # max byte size for SO_SNDBUF
-        sndbuf: nil,
-        # max byte size for SO_RCVBUF
-        rcvbuf: nil
-      }
+        socket_failure_delay: 0.1
+      }.freeze
 
       def initialize(attribs, options = {})
-        @hostname, @port, @weight, @socket_type = ServerConfigParser.parse(attribs)
-        @fail_count = 0
-        @down_at = nil
-        @last_down_at = nil
+        @hostname, @port, @weight, @socket_type, options = ServerConfigParser.parse(attribs, options)
         @options = DEFAULTS.merge(options)
-        @value_compressor = ValueCompressor.new(@options)
+        @value_marshaller = ValueMarshaller.new(@options)
+        @response_processor = ResponseProcessor.new(self, @value_marshaller)
+        @response_buffer = ResponseBuffer.new(self, @response_processor)
+
+        reset_down_info
         @sock = nil
-        @msg = nil
-        @error = nil
         @pid = nil
-        @inprogress = nil
+        @request_in_progress = false
+      end
+
+      def response_buffer
+        @response_buffer ||= ResponseBuffer.new(self, @response_processor)
       end
 
       def name
@@ -56,33 +61,50 @@ module Dalli
         end
       end
 
-      # Chokepoint method for instrumentation
-      def request(op, *args)
-        verify_state
-        raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}. If you are sure it is running, ensure memcached version is > 1.4." unless alive?
+      # Chokepoint method for error handling and ensuring liveness
+      def request(opkey, *args)
+        verify_state(opkey)
+        # The alive? call has the side effect of connecting the underlying
+        # socket if it is not connected, or there's been a disconnect
+        # because of timeout or other error.  Method raises an error
+        # if it can't connect
+        raise_memcached_down_err unless alive?
+
         begin
-          send(op, *args)
-        rescue Dalli::MarshalError => ex
-          Dalli.logger.error "Marshalling error for key '#{args.first}': #{ex.message}"
-          Dalli.logger.error "You are trying to cache a Ruby object which cannot be serialized to memcached."
+          send(opkey, *args)
+        rescue Dalli::MarshalError => e
+          log_marshall_err(args.first, e)
           raise
         rescue Dalli::DalliError, Dalli::NetworkError, Dalli::ValueOverMaxSize, Timeout::Error
           raise
-        rescue => ex
-          Dalli.logger.error "Unexpected exception during Dalli request: #{ex.class.name}: #{ex.message}"
-          Dalli.logger.error ex.backtrace.join("\n\t")
+        rescue StandardError => e
+          log_unexpected_err(e)
           down!
         end
       end
 
+      def raise_memcached_down_err
+        raise Dalli::NetworkError,
+              "#{name} is down: #{@error} #{@msg}. If you are sure it is running, "\
+              "ensure memcached version is > #{::Dalli::MIN_SUPPORTED_MEMCACHED_VERSION}."
+      end
+
+      def log_marshall_err(key, err)
+        Dalli.logger.error "Marshalling error for key '#{key}': #{err.message}"
+        Dalli.logger.error 'You are trying to cache a Ruby object which cannot be serialized to memcached.'
+      end
+
+      def log_unexpected_err(err)
+        Dalli.logger.error "Unexpected exception during Dalli request: #{err.class.name}: #{err.message}"
+        Dalli.logger.error err.backtrace.join("\n\t")
+      end
+
+      # The socket connection to the underlying server is initialized as a side
+      # effect of this call.  In fact, this is the ONLY place where that
+      # socket connection is initialized.
       def alive?
         return true if @sock
-
-        if @last_down_at && @last_down_at + options[:down_retry_delay] >= Time.now
-          time = @last_down_at + options[:down_retry_delay] - Time.now
-          Dalli.logger.debug { "down_retry_delay not reached for #{name} (%.3f seconds left)" % time }
-          return false
-        end
+        return false unless reconnect_down_server?
 
         connect
         !!@sock
@@ -90,128 +112,203 @@ module Dalli
         false
       end
 
+      def reconnect_down_server?
+        return true unless @last_down_at
+
+        time_to_next_reconnect = @last_down_at + options[:down_retry_delay] - Time.now
+        return true unless time_to_next_reconnect.positive?
+
+        Dalli.logger.debug do
+          format('down_retry_delay not reached for %<name>s (%<time>.3f seconds left)', name: name,
+                                                                                        time: time_to_next_reconnect)
+        end
+        false
+      end
+
+      # Closes the underlying socket and cleans up
+      # socket state.
       def close
         return unless @sock
+
         begin
           @sock.close
-        rescue
+        rescue StandardError
           nil
         end
         @sock = nil
         @pid = nil
-        @inprogress = false
+        abort_request!
       end
 
-      def lock!
-      end
+      def lock!; end
 
-      def unlock!
-      end
-
-      def serializer
-        @options[:serializer]
-      end
+      def unlock!; end
 
       # Start reading key/value pairs from this connection. This is usually called
       # after a series of GETKQ commands. A NOOP is sent, and the server begins
       # flushing responses for kv pairs that were found.
       #
       # Returns nothing.
-      def multi_response_start
-        verify_state
+      def pipeline_response_start
+        verify_state(:getkq)
         write_noop
-        @multi_buffer = +""
-        @position = 0
-        @inprogress = true
+        response_buffer.reset
+        start_request!
+      end
+
+      # Did the last call to #pipeline_response_start complete successfully?
+      def pipeline_response_completed?
+        response_buffer.completed?
       end
 
-      # Did the last call to #multi_response_start complete successfully?
-      def multi_response_completed?
-        @multi_buffer.nil?
+      def pipeline_response(bytes_to_advance = 0)
+        response_buffer.process_single_response(bytes_to_advance)
+      end
+
+      def reconnect_on_pipeline_complete!
+        reconnect! 'multi_response has completed' if pipeline_response_completed?
       end
 
       # Attempt to receive and parse as many key/value pairs as possible
-      # from this server. After #multi_response_start, this should be invoked
+      # from this server. After #pipeline_response_start, this should be invoked
       # repeatedly whenever this server's socket is readable until
-      # #multi_response_completed?.
+      # #pipeline_response_completed?.
       #
       # Returns a Hash of kv pairs received.
-      def multi_response_nonblock
-        reconnect! "multi_response has completed" if @multi_buffer.nil?
-
-        @multi_buffer << @sock.read_available
-        buf = @multi_buffer
-        pos = @position
+      def process_outstanding_pipeline_requests
+        reconnect_on_pipeline_complete!
         values = {}
 
-        while buf.bytesize - pos >= 24
-          header = buf.slice(pos, 24)
-          (key_length, _, body_length, cas) = header.unpack(KV_HEADER)
+        response_buffer.read
 
-          if key_length == 0
-            # all done!
-            @multi_buffer = nil
-            @position = nil
-            @inprogress = false
+        bytes_to_advance, status, key, value, cas = pipeline_response
+        # Loop while we have at least a complete header in the buffer
+        while bytes_to_advance.positive?
+          # If the status and key length are both zero, then this is the response
+          # to the noop at the end of the pipeline
+          if status.zero? && key.nil?
+            finish_pipeline
             break
+          end
 
-          elsif buf.bytesize - pos >= 24 + body_length
-            flags = buf.slice(pos + 24, 4).unpack1("N")
-            key = buf.slice(pos + 24 + 4, key_length)
-            value = buf.slice(pos + 24 + 4 + key_length, body_length - key_length - 4) if body_length - key_length - 4 > 0
-
-            pos = pos + 24 + body_length
-
-            begin
-              values[key] = [deserialize(value, flags), cas]
-            rescue DalliError
-            end
+          # If the status is zero and the key len is positive, then this is a
+          # getkq response with a value that we want to set in the response hash
+          values[key] = [value, cas] unless key.nil?
 
-          else
-            # not enough data yet, wait for more
-            break
-          end
+          # Get the next set of bytes from the buffer
+          bytes_to_advance, status, key, value, cas = pipeline_response(bytes_to_advance)
         end
-        @position = pos
 
         values
       rescue SystemCallError, Timeout::Error, EOFError => e
         failure!(e)
       end
 
-      # Abort an earlier #multi_response_start. Used to signal an external
+      def read_nonblock
+        @sock.read_available
+      end
+
+      # Called after the noop response is received at the end of a set
+      # of pipelined gets
+      def finish_pipeline
+        response_buffer.clear
+        finish_request!
+      end
+
+      # Abort an earlier #pipeline_response_start. Used to signal an external
       # timeout. The underlying socket is disconnected, and the exception is
       # swallowed.
       #
       # Returns nothing.
-      def multi_response_abort
-        @multi_buffer = nil
-        @position = nil
-        @inprogress = false
-        failure!(RuntimeError.new("External timeout"))
+      def pipeline_response_abort
+        response_buffer.clear
+        abort_request!
+        return true unless @sock
+
+        failure!(RuntimeError.new('External timeout'))
       rescue NetworkError
         true
       end
 
+      def read(count)
+        start_request!
+        data = @sock.readfull(count)
+        finish_request!
+        data
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        failure!(e)
+      end
+
+      def write(bytes)
+        start_request!
+        result = @sock.write(bytes)
+        finish_request!
+        result
+      rescue SystemCallError, Timeout::Error => e
+        failure!(e)
+      end
+
+      def connected?
+        !@sock.nil?
+      end
+
+      def socket_timeout
+        @socket_timeout ||= @options[:socket_timeout]
+      end
+
       # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
 
       private
 
-      def verify_state
-        failure!(RuntimeError.new("Already writing to socket")) if @inprogress
-        if @pid && @pid != Process.pid
-          message = "Fork detected, re-connecting child process..."
-          Dalli.logger.info { message }
-          reconnect! message
-        end
+      def request_in_progress?
+        @request_in_progress
+      end
+
+      def start_request!
+        @request_in_progress = true
+      end
+
+      def finish_request!
+        @request_in_progress = false
+      end
+
+      def abort_request!
+        @request_in_progress = false
+      end
+
+      def verify_state(opkey)
+        failure!(RuntimeError.new('Already writing to socket')) if request_in_progress?
+        reconnect_on_fork if fork_detected?
+        verify_allowed_multi!(opkey) if multi?
+      end
+
+      def fork_detected?
+        @pid && @pid != Process.pid
       end
 
+      ALLOWED_MULTI_OPS = %i[add addq delete deleteq replace replaceq set setq noop].freeze
+      def verify_allowed_multi!(opkey)
+        return if ALLOWED_MULTI_OPS.include?(opkey)
+
+        raise Dalli::NotPermittedMultiOpError, "The operation #{opkey} is not allowed in a multi block."
+      end
+
+      def reconnect_on_fork
+        message = 'Fork detected, re-connecting child process...'
+        Dalli.logger.info { message }
+        reconnect! message
+      end
+
+      # Marks the server instance as needing reconnect.  Raises a
+      # Dalli::NetworkError with the specified message.  Calls close
+      # to clean up socket state
       def reconnect!(message)
         close
         sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
         raise Dalli::NetworkError, message
       end
 
+      # Raises Dalli::NetworkError
       def failure!(exception)
         message = "#{name} failed (count: #{@fail_count}) #{exception.class}: #{exception.message}"
         Dalli.logger.warn { message }
@@ -220,34 +317,47 @@ module Dalli
         if @fail_count >= options[:socket_max_failures]
           down!
         else
-          reconnect! "Socket operation failed, retrying..."
+          reconnect! 'Socket operation failed, retrying...'
         end
       end
 
+      # Marks the server instance as down.  Updates the down_at state
+      # and raises an Dalli::NetworkError that includes the underlying
+      # error in the message.  Calls close to clean up socket state
       def down!
         close
+        log_down_detected
 
+        @error = $ERROR_INFO&.class&.name
+        @msg ||= $ERROR_INFO&.message
+        raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
+      end
+
+      def log_down_detected
         @last_down_at = Time.now
 
         if @down_at
           time = Time.now - @down_at
-          Dalli.logger.debug { "#{name} is still down (for %.3f seconds now)" % time }
+          Dalli.logger.debug { format('%<name>s is still down (for %<time>.3f seconds now)', name: name, time: time) }
         else
           @down_at = @last_down_at
-          Dalli.logger.warn { "#{name} is down" }
+          Dalli.logger.warn("#{name} is down")
         end
+      end
 
-        @error = $!&.class&.name
-        @msg ||= $!&.message
-        raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
+      def log_up_detected
+        return unless @down_at
+
+        time = Time.now - @down_at
+        Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
       end
 
       def up!
-        if @down_at
-          time = Time.now - @down_at
-          Dalli.logger.warn { "#{name} is back (downtime was %.3f seconds)" % time }
-        end
+        log_up_detected
+        reset_down_info
+      end
 
+      def reset_down_info
         @fail_count = 0
         @down_at = nil
         @last_down_at = nil
@@ -256,99 +366,106 @@ module Dalli
       end
 
       def multi?
-        Thread.current[:dalli_multi]
+        Thread.current[::Dalli::MULTI_KEY]
+      end
+
+      def cache_nils?(opts)
+        return false unless opts.is_a?(Hash)
+
+        opts[:cache_nils] ? true : false
       end
 
       def get(key, options = nil)
-        req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
+        req = RequestFormatter.standard_request(opkey: :get, key: key)
         write(req)
-        generic_response(true, !!(options && options.is_a?(Hash) && options[:cache_nils]))
+        @response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
       end
 
-      def send_multiget(keys)
-        req = +""
+      def pipelined_get(keys)
+        req = +''
         keys.each do |key|
-          req << [REQUEST, OPCODES[:getkq], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:getkq])
+          req << RequestFormatter.standard_request(opkey: :getkq, key: key)
         end
-        # Could send noop here instead of in multi_response_start
+        # Could send noop here instead of in pipeline_response_start
         write(req)
       end
 
       def set(key, value, ttl, cas, options)
-        (value, flags) = serialize(key, value, options)
-        ttl = TtlSanitizer.sanitize(ttl)
-
-        guard_max_value(key, value)
-
-        req = [REQUEST, OPCODES[multi? ? :setq : :set], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:set])
-        write(req)
-        cas_response unless multi?
+        opkey = multi? ? :setq : :set
+        process_value_req(opkey, key, value, ttl, cas, options)
       end
 
       def add(key, value, ttl, options)
-        (value, flags) = serialize(key, value, options)
-        ttl = TtlSanitizer.sanitize(ttl)
-
-        guard_max_value(key, value)
-
-        req = [REQUEST, OPCODES[multi? ? :addq : :add], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:add])
-        write(req)
-        cas_response unless multi?
+        opkey = multi? ? :addq : :add
+        cas = 0
+        process_value_req(opkey, key, value, ttl, cas, options)
       end
 
       def replace(key, value, ttl, cas, options)
-        (value, flags) = serialize(key, value, options)
-        ttl = TtlSanitizer.sanitize(ttl)
+        opkey = multi? ? :replaceq : :replace
+        process_value_req(opkey, key, value, ttl, cas, options)
+      end
 
-        guard_max_value(key, value)
+      # rubocop:disable Metrics/ParameterLists
+      def process_value_req(opkey, key, value, ttl, cas, options)
+        (value, bitflags) = @value_marshaller.store(key, value, options)
+        ttl = TtlSanitizer.sanitize(ttl)
 
-        req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:replace])
+        req = RequestFormatter.standard_request(opkey: opkey, key: key,
+                                                value: value, bitflags: bitflags,
+                                                ttl: ttl, cas: cas)
         write(req)
-        cas_response unless multi?
+        @response_processor.cas_response unless multi?
       end
+      # rubocop:enable Metrics/ParameterLists
 
       def delete(key, cas)
-        req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, cas, key].pack(FORMAT[:delete])
+        opkey = multi? ? :deleteq : :delete
+        req = RequestFormatter.standard_request(opkey: opkey, key: key, cas: cas)
         write(req)
-        generic_response unless multi?
+        @response_processor.generic_response unless multi?
       end
 
-      def flush(ttl)
-        req = [REQUEST, OPCODES[:flush], 0, 4, 0, 0, 4, 0, 0, 0].pack(FORMAT[:flush])
+      def flush(ttl = 0)
+        req = RequestFormatter.standard_request(opkey: :flush, ttl: ttl)
         write(req)
-        generic_response
+        @response_processor.generic_response
       end
 
-      def decr_incr(opcode, key, count, ttl, default)
-        expiry = default ? TtlSanitizer.sanitize(ttl) : 0xFFFFFFFF
-        default ||= 0
-        (h, l) = split(count)
-        (dh, dl) = split(default)
-        req = [REQUEST, OPCODES[opcode], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[opcode])
-        write(req)
-        body = generic_response
-        body ? body.unpack1("Q>") : body
+      # This allows us to special case a nil initial value, and
+      # handle it differently than a zero.  This special value
+      # for expiry causes memcached to return a not found
+      # if the key doesn't already exist, rather than
+      # setting the initial value
+      NOT_FOUND_EXPIRY = 0xFFFFFFFF
+
+      def decr_incr(opkey, key, count, ttl, initial)
+        expiry = initial ? TtlSanitizer.sanitize(ttl) : NOT_FOUND_EXPIRY
+        initial ||= 0
+        write(RequestFormatter.decr_incr_request(opkey: opkey, key: key,
+                                                 count: count, initial: initial, expiry: expiry))
+        @response_processor.decr_incr_response
       end
 
-      def decr(key, count, ttl, default)
-        decr_incr :decr, key, count, ttl, default
+      def decr(key, count, ttl, initial)
+        decr_incr :decr, key, count, ttl, initial
       end
 
-      def incr(key, count, ttl, default)
-        decr_incr :incr, key, count, ttl, default
+      def incr(key, count, ttl, initial)
+        decr_incr :incr, key, count, ttl, initial
       end
 
-      def write_append_prepend(opcode, key, value)
-        write_generic [REQUEST, OPCODES[opcode], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[opcode])
+      def write_append_prepend(opkey, key, value)
+        write_generic RequestFormatter.standard_request(opkey: opkey, key: key, value: value)
       end
 
       def write_generic(bytes)
         write(bytes)
-        generic_response
+        @response_processor.generic_response
       end
 
       def write_noop
-        req = [REQUEST, OPCODES[:noop], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
+        req = RequestFormatter.standard_request(opkey: :noop)
         write(req)
       end
 
@@ -356,7 +473,7 @@ module Dalli
       # We need to read all the responses at once.
       def noop
         write_noop
-        multi_response
+        @response_processor.multi_with_keys_response
       end
 
       def append(key, value)
@@ -367,188 +484,36 @@ module Dalli
         write_append_prepend :prepend, key, value
       end
 
-      def stats(info = "")
-        req = [REQUEST, OPCODES[:stat], info.bytesize, 0, 0, 0, info.bytesize, 0, 0, info].pack(FORMAT[:stat])
+      def stats(info = '')
+        req = RequestFormatter.standard_request(opkey: :stat, key: info)
         write(req)
-        keyvalue_response
+        @response_processor.multi_with_keys_response
       end
 
       def reset_stats
-        write_generic [REQUEST, OPCODES[:stat], "reset".bytesize, 0, 0, 0, "reset".bytesize, 0, 0, "reset"].pack(FORMAT[:stat])
+        write_generic RequestFormatter.standard_request(opkey: :stat, key: 'reset')
       end
 
       def cas(key)
-        req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
+        req = RequestFormatter.standard_request(opkey: :get, key: key)
         write(req)
-        data_cas_response
+        @response_processor.data_cas_response
       end
 
       def version
-        write_generic [REQUEST, OPCODES[:version], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
+        write_generic RequestFormatter.standard_request(opkey: :version)
       end
 
       def touch(key, ttl)
         ttl = TtlSanitizer.sanitize(ttl)
-        write_generic [REQUEST, OPCODES[:touch], key.bytesize, 4, 0, 0, key.bytesize + 4, 0, 0, ttl, key].pack(FORMAT[:touch])
+        write_generic RequestFormatter.standard_request(opkey: :touch, key: key, ttl: ttl)
       end
 
       def gat(key, ttl, options = nil)
         ttl = TtlSanitizer.sanitize(ttl)
-        req = [REQUEST, OPCODES[:gat], key.bytesize, 4, 0, 0, key.bytesize + 4, 0, 0, ttl, key].pack(FORMAT[:gat])
+        req = RequestFormatter.standard_request(opkey: :gat, key: key, ttl: ttl)
         write(req)
-        generic_response(true, !!(options && options.is_a?(Hash) && options[:cache_nils]))
-      end
-
-      # https://www.hjp.at/zettel/m/memcached_flags.rxml
-      # Looks like most clients use bit 0 to indicate native language serialization
-      FLAG_SERIALIZED = 0x1
-
-      def serialize(key, value, options = nil)
-        marshalled = false
-        value = if options && options[:raw]
-          value.to_s
-        else
-          marshalled = true
-          begin
-            serializer.dump(value)
-          rescue Timeout::Error => e
-            raise e
-          rescue => ex
-            # Marshalling can throw several different types of generic Ruby exceptions.
-            # Convert to a specific exception so we can special case it higher up the stack.
-            exc = Dalli::MarshalError.new(ex.message)
-            exc.set_backtrace ex.backtrace
-            raise exc
-          end
-        end
-
-        bitflags = 0
-        value, bitflags = @value_compressor.store(value, options, bitflags)
-
-        bitflags |= FLAG_SERIALIZED if marshalled
-        [value, bitflags]
-      end
-
-      def deserialize(value, flags)
-        value = @value_compressor.retrieve(value, flags)
-        value = serializer.load(value) if (flags & FLAG_SERIALIZED) != 0
-        value
-      rescue TypeError
-        raise unless /needs to have method `_load'|exception class\/object expected|instance of IO needed|incompatible marshal file format/.match?($!.message)
-        raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
-      rescue ArgumentError
-        raise unless /undefined class|marshal data too short/.match?($!.message)
-        raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
-      rescue NameError
-        raise unless /uninitialized constant/.match?($!.message)
-        raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
-      end
-
-      def data_cas_response
-        (extras, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
-        data = read(count) if count > 0
-        if status == 1
-          nil
-        elsif status != 0
-          raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
-        elsif data
-          flags = data[0...extras].unpack1("N")
-          value = data[extras..-1]
-          data = deserialize(value, flags)
-        end
-        [data, cas]
-      end
-
-      CAS_HEADER = "@4CCnNNQ"
-      NORMAL_HEADER = "@4CCnN"
-      KV_HEADER = "@2n@6nN@16Q"
-
-      def guard_max_value(key, value)
-        return if value.bytesize <= @options[:value_max_bytes]
-
-        message = "Value for #{key} over max size: #{@options[:value_max_bytes]} <= #{value.bytesize}"
-        raise Dalli::ValueOverMaxSize, message
-      end
-
-      # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
-      class NilObject; end
-      NOT_FOUND = NilObject.new
-
-      def generic_response(unpack = false, cache_nils = false)
-        (extras, _, status, count) = read_header.unpack(NORMAL_HEADER)
-        data = read(count) if count > 0
-        if status == 1
-          cache_nils ? NOT_FOUND : nil
-        elsif status == 2 || status == 5
-          false # Not stored, normal status for add operation
-        elsif status != 0
-          raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
-        elsif data
-          flags = data.byteslice(0, extras).unpack1("N")
-          value = data.byteslice(extras, data.bytesize - extras)
-          unpack ? deserialize(value, flags) : value
-        else
-          true
-        end
-      end
-
-      def cas_response
-        (_, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
-        read(count) if count > 0 # this is potential data that we don't care about
-        if status == 1
-          nil
-        elsif status == 2 || status == 5
-          false # Not stored, normal status for add operation
-        elsif status != 0
-          raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
-        else
-          cas
-        end
-      end
-
-      def keyvalue_response
-        hash = {}
-        loop do
-          (key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
-          return hash if key_length == 0
-          key = read(key_length)
-          value = read(body_length - key_length) if body_length - key_length > 0
-          hash[key] = value
-        end
-      end
-
-      def multi_response
-        hash = {}
-        loop do
-          (key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
-          return hash if key_length == 0
-          flags = read(4).unpack1("N")
-          key = read(key_length)
-          value = read(body_length - key_length - 4) if body_length - key_length - 4 > 0
-          hash[key] = deserialize(value, flags)
-        end
-      end
-
-      def write(bytes)
-        @inprogress = true
-        result = @sock.write(bytes)
-        @inprogress = false
-        result
-      rescue SystemCallError, Timeout::Error => e
-        failure!(e)
-      end
-
-      def read(count)
-        @inprogress = true
-        data = @sock.readfull(count)
-        @inprogress = false
-        data
-      rescue SystemCallError, Timeout::Error, EOFError => e
-        failure!(e)
-      end
-
-      def read_header
-        read(24) || raise(Dalli::NetworkError, "No response")
+        @response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
       end
 
       def connect
@@ -556,13 +521,9 @@ module Dalli
 
         begin
           @pid = Process.pid
-          @sock = if socket_type == :unix
-            Dalli::Socket::UNIX.open(hostname, self, options)
-          else
-            Dalli::Socket::TCP.open(hostname, port, self, options)
-          end
-          sasl_authentication if need_auth?
-          @version = version # trigger actual connect
+          @sock = memcached_socket
+          authenticate_connection if require_auth?
+          @version = version # Connect socket if not authed
           up!
         rescue Dalli::DalliError # SASL auth failure
           raise
@@ -572,132 +533,27 @@ module Dalli
         end
       end
 
-      def split(n)
-        [n >> 32, 0xFFFFFFFF & n]
-      end
-
-      REQUEST = 0x80
-      RESPONSE = 0x81
-
-      # Response codes taken from:
-      # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
-      RESPONSE_CODES = {
-        0 => "No error",
-        1 => "Key not found",
-        2 => "Key exists",
-        3 => "Value too large",
-        4 => "Invalid arguments",
-        5 => "Item not stored",
-        6 => "Incr/decr on a non-numeric value",
-        7 => "The vbucket belongs to another server",
-        8 => "Authentication error",
-        9 => "Authentication continue",
-        0x20 => "Authentication required",
-        0x81 => "Unknown command",
-        0x82 => "Out of memory",
-        0x83 => "Not supported",
-        0x84 => "Internal error",
-        0x85 => "Busy",
-        0x86 => "Temporary failure"
-      }
-
-      OPCODES = {
-        get: 0x00,
-        set: 0x01,
-        add: 0x02,
-        replace: 0x03,
-        delete: 0x04,
-        incr: 0x05,
-        decr: 0x06,
-        flush: 0x08,
-        noop: 0x0A,
-        version: 0x0B,
-        getkq: 0x0D,
-        append: 0x0E,
-        prepend: 0x0F,
-        stat: 0x10,
-        setq: 0x11,
-        addq: 0x12,
-        replaceq: 0x13,
-        deleteq: 0x14,
-        incrq: 0x15,
-        decrq: 0x16,
-        auth_negotiation: 0x20,
-        auth_request: 0x21,
-        auth_continue: 0x22,
-        touch: 0x1C,
-        gat: 0x1D
-      }
-
-      HEADER = "CCnCCnNNQ"
-      OP_FORMAT = {
-        get: "a*",
-        set: "NNa*a*",
-        add: "NNa*a*",
-        replace: "NNa*a*",
-        delete: "a*",
-        incr: "NNNNNa*",
-        decr: "NNNNNa*",
-        flush: "N",
-        noop: "",
-        getkq: "a*",
-        version: "",
-        stat: "a*",
-        append: "a*a*",
-        prepend: "a*a*",
-        auth_request: "a*a*",
-        auth_continue: "a*a*",
-        touch: "Na*",
-        gat: "Na*"
-      }
-      FORMAT = OP_FORMAT.each_with_object({}) { |(k, v), memo| memo[k] = HEADER + v; }
-
-      #######
-      # SASL authentication support for NorthScale
-      #######
-
-      def need_auth?
-        @options[:username] || ENV["MEMCACHE_USERNAME"]
+      def memcached_socket
+        if socket_type == :unix
+          Dalli::Socket::UNIX.open(hostname, options)
+        else
+          Dalli::Socket::TCP.open(hostname, port, options)
+        end
+      end
+
+      def require_auth?
+        !username.nil?
       end
 
       def username
-        @options[:username] || ENV["MEMCACHE_USERNAME"]
+        @options[:username] || ENV['MEMCACHE_USERNAME']
       end
 
       def password
-        @options[:password] || ENV["MEMCACHE_PASSWORD"]
+        @options[:password] || ENV['MEMCACHE_PASSWORD']
       end
 
-      def sasl_authentication
-        Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
-
-        # negotiate
-        req = [REQUEST, OPCODES[:auth_negotiation], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
-        write(req)
-
-        (extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
-        raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
-        content = read(count).tr("\u0000", " ")
-        return Dalli.logger.debug("Authentication not required/supported by server") if status == 0x81
-        mechanisms = content.split(" ")
-        raise NotImplementedError, "Dalli only supports the PLAIN authentication mechanism" unless mechanisms.include?("PLAIN")
-
-        # request
-        mechanism = "PLAIN"
-        msg = "\x0#{username}\x0#{password}"
-        req = [REQUEST, OPCODES[:auth_request], mechanism.bytesize, 0, 0, 0, mechanism.bytesize + msg.bytesize, 0, 0, mechanism, msg].pack(FORMAT[:auth_request])
-        write(req)
-
-        (extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
-        raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
-        content = read(count)
-        return Dalli.logger.info("Dalli/SASL: #{content}") if status == 0
-
-        raise Dalli::DalliError, "Error authenticating: #{status}" unless status == 0x21
-        raise NotImplementedError, "No two-step authentication mechanisms supported"
-        # (step, msg) = sasl.receive('challenge', content)
-        # raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
-      end
+      include SaslAuthentication
     end
   end
 end