dalli 1.1.4 → 2.7.11

data/lib/dalli/compressor.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+require 'zlib'
+require 'stringio'
+
+module Dalli
+  class Compressor
+    def self.compress(data)
+      Zlib::Deflate.deflate(data)
+    end
+
+    def self.decompress(data)
+      Zlib::Inflate.inflate(data)
+    end
+  end
+
+  class GzipCompressor
+    def self.compress(data)
+      io = StringIO.new(String.new(""), "w")
+      gz = Zlib::GzipWriter.new(io)
+      gz.write(data)
+      gz.close
+      io.string
+    end
+
+    def self.decompress(data)
+      io = StringIO.new(data, "rb")
+      Zlib::GzipReader.new(io).read
+    end
+  end
+end

data/lib/dalli/options.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'thread'
 require 'monitor'
 
@@ -31,6 +32,24 @@ module Dalli
       end
     end
 
+    def multi_response_start
+      @lock.synchronize do
+        super
+      end
+    end
+
+    def multi_response_nonblock
+      @lock.synchronize do
+        super
+      end
+    end
+
+    def multi_response_abort
+      @lock.synchronize do
+        super
+      end
+    end
+
     def lock!
       @lock.mon_enter
     end

data/lib/dalli/railtie.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+module Dalli
+  class Railtie < ::Rails::Railtie
+    config.before_configuration do
+      config.cache_store = :dalli_store
+    end
+  end
+end

data/lib/dalli/ring.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'digest/sha1'
 require 'zlib'
 
@@ -15,12 +16,12 @@ module Dalli
         continuum = []
         servers.each do |server|
           entry_count_for(server, servers.size, total_weight).times do |idx|
-            hash = Digest::SHA1.hexdigest("#{server.hostname}:#{server.port}:#{idx}")
+            hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
             value = Integer("0x#{hash[0..7]}")
             continuum << Dalli::Ring::Entry.new(value, server)
           end
         end
-        @continuum = continuum.sort { |a, b| a.value <=> b.value }
+        @continuum = continuum.sort_by(&:value)
       end
 
       threadsafe! unless options[:threadsafe] == false
@@ -31,7 +32,7 @@ module Dalli
       if @continuum
         hkey = hash_for(key)
         20.times do |try|
-          entryidx = self.class.binary_search(@continuum, hkey)
+          entryidx = binary_search(@continuum, hkey)
           server = @continuum[entryidx].server
           return server if server.alive?
           break unless @failover
@@ -46,11 +47,11 @@ module Dalli
     end
 
     def lock
-      @servers.each { |s| s.lock! }
+      @servers.each(&:lock!)
       begin
         return yield
       ensure
-        @servers.each { |s| s.unlock! }
+        @servers.each(&:unlock!)
       end
     end
 
@@ -70,25 +71,61 @@ module Dalli
       ((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
     end
 
-    # Find the closest index in the Ring with value <= the given value
-    def self.binary_search(ary, value)
-      upper = ary.size - 1
-      lower = 0
-      idx = 0
-
-      while (lower <= upper) do
-        idx = (lower + upper) / 2
-        comp = ary[idx].value <=> value
-
-        if comp == 0
-          return idx
-        elsif comp > 0
-          upper = idx - 1
-        else
-          lower = idx + 1
+    # Native extension to perform the binary search within the continuum
+    # space.  Fallback to a pure Ruby version if the compilation doesn't work.
+    # optional for performance and only necessary if you are using multiple
+    # memcached servers.
+    begin
+      require 'inline'
+      inline do |builder|
+        builder.c <<-EOM
+        int binary_search(VALUE ary, unsigned int r) {
+            long upper = RARRAY_LEN(ary) - 1;
+            long lower = 0;
+            long idx = 0;
+            ID value = rb_intern("value");
+            VALUE continuumValue;
+            unsigned int l;
+
+            while (lower <= upper) {
+                idx = (lower + upper) / 2;
+
+                continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
+                l = NUM2UINT(continuumValue);
+                if (l == r) {
+                    return idx;
+                }
+                else if (l > r) {
+                    upper = idx - 1;
+                }
+                else {
+                    lower = idx + 1;
+                }
+            }
+            return upper;
+        }
+        EOM
+      end
+    rescue LoadError
+      # Find the closest index in the Ring with value <= the given value
+      def binary_search(ary, value)
+        upper = ary.size - 1
+        lower = 0
+
+        while (lower <= upper) do
+          idx = (lower + upper) / 2
+          comp = ary[idx].value <=> value
+
+          if comp == 0
+            return idx
+          elsif comp > 0
+            upper = idx - 1
+          else
+            lower = idx + 1
+          end
         end
+        upper
       end
-      return upper
     end
 
     class Entry

data/lib/dalli/server.rb

@@ -1,6 +1,6 @@
+# frozen_string_literal: true
 require 'socket'
 require 'timeout'
-require 'zlib'
 
 module Dalli
   class Server
@@ -8,10 +8,14 @@ module Dalli
     attr_accessor :port
     attr_accessor :weight
     attr_accessor :options
+    attr_reader :sock
+    attr_reader :socket_type  # possible values: :unix, :tcp
 
+    DEFAULT_PORT = 11211
+    DEFAULT_WEIGHT = 1
     DEFAULTS = {
       # seconds between trying to contact a remote server
-      :down_retry_delay => 1,
+      :down_retry_delay => 60,
       # connect/read/write timeout for socket operations
       :socket_timeout => 0.5,
       # times a socket operation may fail before considering the server dead
@@ -20,42 +24,59 @@ module Dalli
       :socket_failure_delay => 0.01,
       # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
       :value_max_bytes => 1024 * 1024,
+      # surpassing value_max_bytes either warns (false) or throws (true)
+      :error_when_over_max_size => false,
+      :compressor => Compressor,
+      # min byte size to attempt compression
+      :compression_min_size => 1024,
+      # max byte size for compression
+      :compression_max_size => false,
+      :serializer => Marshal,
       :username => nil,
       :password => nil,
-      :async => false,
+      :keepalive => true,
+      # max byte size for SO_SNDBUF
+      :sndbuf => nil,
+      # max byte size for SO_RCVBUF
+      :rcvbuf => nil
     }
 
     def initialize(attribs, options = {})
-      (@hostname, @port, @weight) = attribs.split(':')
-      @port ||= 11211
-      @port = Integer(@port)
-      @weight ||= 1
-      @weight = Integer(@weight)
+      @hostname, @port, @weight, @socket_type = parse_hostname(attribs)
       @fail_count = 0
       @down_at = nil
       @last_down_at = nil
       @options = DEFAULTS.merge(options)
       @sock = nil
       @msg = nil
+      @error = nil
+      @pid = nil
+      @inprogress = nil
+    end
+
+    def name
+      if socket_type == :unix
+        hostname
+      else
+        "#{hostname}:#{port}"
+      end
     end
 
     # Chokepoint method for instrumentation
     def request(op, *args)
-      raise Dalli::NetworkError, "#{hostname}:#{port} is down: #{@error} #{@msg}" unless alive?
+      verify_state
+      raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}. If you are sure it is running, ensure memcached version is > 1.4." unless alive?
       begin
         send(op, *args)
-      rescue Dalli::NetworkError
-        raise
       rescue Dalli::MarshalError => ex
         Dalli.logger.error "Marshalling error for key '#{args.first}': #{ex.message}"
         Dalli.logger.error "You are trying to cache a Ruby object which cannot be serialized to memcached."
         Dalli.logger.error ex.backtrace.join("\n\t")
         false
-      rescue Dalli::DalliError
+      rescue Dalli::DalliError, Dalli::NetworkError, Dalli::ValueOverMaxSize, Timeout::Error
         raise
       rescue => ex
-        Dalli.logger.error "Unexpected exception in Dalli: #{ex.class.name}: #{ex.message}"
-        Dalli.logger.error "This is a bug in Dalli, please enter an issue in Github if it does not already exist."
+        Dalli.logger.error "Unexpected exception during Dalli request: #{ex.class.name}: #{ex.message}"
         Dalli.logger.error ex.backtrace.join("\n\t")
         down!
       end
@@ -66,7 +87,7 @@ module Dalli
 
       if @last_down_at && @last_down_at + options[:down_retry_delay] >= Time.now
         time = @last_down_at + options[:down_retry_delay] - Time.now
-        Dalli.logger.debug { "down_retry_delay not reached for #{hostname}:#{port} (%.3f seconds left)" % time }
+        Dalli.logger.debug { "down_retry_delay not reached for #{name} (%.3f seconds left)" % time }
         return false
       end
 
@@ -80,6 +101,8 @@ module Dalli
       return unless @sock
       @sock.close rescue nil
       @sock = nil
+      @pid = nil
+      @inprogress = false
     end
 
     def lock!
@@ -88,24 +111,123 @@ module Dalli
     def unlock!
     end
 
+    def serializer
+      @options[:serializer]
+    end
+
+    def compressor
+      @options[:compressor]
+    end
+
+    # Start reading key/value pairs from this connection. This is usually called
+    # after a series of GETKQ commands. A NOOP is sent, and the server begins
+    # flushing responses for kv pairs that were found.
+    #
+    # Returns nothing.
+    def multi_response_start
+      verify_state
+      write_noop
+      @multi_buffer = String.new('')
+      @position = 0
+      @inprogress = true
+    end
+
+    # Did the last call to #multi_response_start complete successfully?
+    def multi_response_completed?
+      @multi_buffer.nil?
+    end
+
+    # Attempt to receive and parse as many key/value pairs as possible
+    # from this server. After #multi_response_start, this should be invoked
+    # repeatedly whenever this server's socket is readable until
+    # #multi_response_completed?.
+    #
+    # Returns a Hash of kv pairs received.
+    def multi_response_nonblock
+      raise 'multi_response has completed' if @multi_buffer.nil?
+
+      @multi_buffer << @sock.read_available
+      buf = @multi_buffer
+      pos = @position
+      values = {}
+
+      while buf.bytesize - pos >= 24
+        header = buf.slice(pos, 24)
+        (key_length, _, body_length, cas) = header.unpack(KV_HEADER)
+
+        if key_length == 0
+          # all done!
+          @multi_buffer = nil
+          @position = nil
+          @inprogress = false
+          break
+
+        elsif buf.bytesize - pos >= 24 + body_length
+          flags = buf.slice(pos + 24, 4).unpack('N')[0]
+          key = buf.slice(pos + 24 + 4, key_length)
+          value = buf.slice(pos + 24 + 4 + key_length, body_length - key_length - 4) if body_length - key_length - 4 > 0
+
+          pos = pos + 24 + body_length
+
+          begin
+            values[key] = [deserialize(value, flags), cas]
+          rescue DalliError
+          end
+
+        else
+          # not enough data yet, wait for more
+          break
+        end
+      end
+      @position = pos
+
+      values
+    rescue SystemCallError, Timeout::Error, EOFError => e
+      failure!(e)
+    end
+
+    # Abort an earlier #multi_response_start. Used to signal an external
+    # timeout. The underlying socket is disconnected, and the exception is
+    # swallowed.
+    #
+    # Returns nothing.
+    def multi_response_abort
+      @multi_buffer = nil
+      @position = nil
+      @inprogress = false
+      failure!(RuntimeError.new('External timeout'))
+    rescue NetworkError
+      true
+    end
+
     # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
 
     private
 
-    def is_unix_socket?(string)
-      !!(/^\/(.+)$/ =~ string)
+    def verify_state
+      failure!(RuntimeError.new('Already writing to socket')) if @inprogress
+      if @pid && @pid != Process.pid
+        message = 'Fork detected, re-connecting child process...'
+        Dalli.logger.info { message }
+        reconnect! message
+      end
     end
 
-    def failure!
-      Dalli.logger.info { "#{hostname}:#{port} failed (count: #{@fail_count})" }
+    def reconnect!(message)
+      close
+      sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
+      raise Dalli::NetworkError, message
+    end
+
+    def failure!(exception)
+      message = "#{name} failed (count: #{@fail_count}) #{exception.class}: #{exception.message}"
+      Dalli.logger.warn { message }
 
       @fail_count += 1
       if @fail_count >= options[:socket_max_failures]
         down!
       else
-        close
-        sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
-        raise Dalli::NetworkError, "Socket operation failed, retrying..."
+        reconnect! 'Socket operation failed, retrying...'
       end
     end
 
@@ -116,21 +238,21 @@ module Dalli
 
       if @down_at
         time = Time.now - @down_at
-        Dalli.logger.debug { "#{hostname}:#{port} is still down (for %.3f seconds now)" % time }
+        Dalli.logger.debug { "#{name} is still down (for %.3f seconds now)" % time }
       else
         @down_at = @last_down_at
-        Dalli.logger.warn { "#{hostname}:#{port} is down" }
+        Dalli.logger.warn { "#{name} is down" }
       end
 
       @error = $! && $!.class.name
       @msg = @msg || ($! && $!.message && !$!.message.empty? && $!.message)
-      raise Dalli::NetworkError, "#{hostname}:#{port} is down: #{@error} #{@msg}"
+      raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
     end
 
     def up!
       if @down_at
         time = Time.now - @down_at
-        Dalli.logger.warn { "#{hostname}:#{port} is back (downtime was %.3f seconds)" % time }
+        Dalli.logger.warn { "#{name} is back (downtime was %.3f seconds)" % time }
       end
 
       @fail_count = 0
@@ -144,42 +266,56 @@ module Dalli
       Thread.current[:dalli_multi]
     end
 
-    def get(key)
+    def get(key, options=nil)
       req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
       write(req)
-      generic_response(true)
+      generic_response(true, !!(options && options.is_a?(Hash) && options[:cache_nils]))
     end
 
-    def getkq(key)
-      req = [REQUEST, OPCODES[:getkq], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:getkq])
+    def send_multiget(keys)
+      req = String.new("")
+      keys.each do |key|
+        req << [REQUEST, OPCODES[:getkq], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:getkq])
+      end
+      # Could send noop here instead of in multi_response_start
       write(req)
     end
 
     def set(key, value, ttl, cas, options)
       (value, flags) = serialize(key, value, options)
+      ttl = sanitize_ttl(ttl)
 
-      req = [REQUEST, OPCODES[multi? ? :setq : :set], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:set])
-      write(req)
-      generic_response unless multi?
+      guard_max_value(key, value) do
+        req = [REQUEST, OPCODES[multi? ? :setq : :set], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:set])
+        write(req)
+        cas_response unless multi?
+      end
     end
 
     def add(key, value, ttl, options)
       (value, flags) = serialize(key, value, options)
+      ttl = sanitize_ttl(ttl)
 
-      req = [REQUEST, OPCODES[multi? ? :addq : :add], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:add])
-      write(req)
-      generic_response unless multi?
+      guard_max_value(key, value) do
+        req = [REQUEST, OPCODES[multi? ? :addq : :add], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:add])
+        write(req)
+        cas_response unless multi?
+      end
     end
 
-    def replace(key, value, ttl, options)
+    def replace(key, value, ttl, cas, options)
       (value, flags) = serialize(key, value, options)
-      req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:replace])
-      write(req)
-      generic_response unless multi?
+      ttl = sanitize_ttl(ttl)
+
+      guard_max_value(key, value) do
+        req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:replace])
+        write(req)
+        cas_response unless multi?
+      end
     end
 
-    def delete(key)
-      req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:delete])
+    def delete(key, cas)
+      req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, cas, key].pack(FORMAT[:delete])
       write(req)
       generic_response unless multi?
     end
@@ -190,46 +326,52 @@ module Dalli
       generic_response
     end
 
-    def decr(key, count, ttl, default)
-      expiry = default ? ttl : 0xFFFFFFFF
+    def decr_incr(opcode, key, count, ttl, default)
+      expiry = default ? sanitize_ttl(ttl) : 0xFFFFFFFF
       default ||= 0
       (h, l) = split(count)
       (dh, dl) = split(default)
-      req = [REQUEST, OPCODES[:decr], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[:decr])
+      req = [REQUEST, OPCODES[opcode], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[opcode])
       write(req)
       body = generic_response
-      body ? longlong(*body.unpack('NN')) : body
+      body ? body.unpack('Q>').first : body
+    end
+
+    def decr(key, count, ttl, default)
+      decr_incr :decr, key, count, ttl, default
     end
 
     def incr(key, count, ttl, default)
-      expiry = default ? ttl : 0xFFFFFFFF
-      default ||= 0
-      (h, l) = split(count)
-      (dh, dl) = split(default)
-      req = [REQUEST, OPCODES[:incr], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[:incr])
+      decr_incr :incr, key, count, ttl, default
+    end
+
+    def write_append_prepend(opcode, key, value)
+      write_generic [REQUEST, OPCODES[opcode], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[opcode])
+    end
+
+    def write_generic(bytes)
+      write(bytes)
+      generic_response
+    end
+
+    def write_noop
+      req = [REQUEST, OPCODES[:noop], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
       write(req)
-      body = generic_response
-      body ? longlong(*body.unpack('NN')) : body
     end
 
     # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
     # We need to read all the responses at once.
     def noop
-      req = [REQUEST, OPCODES[:noop], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
-      write(req)
+      write_noop
       multi_response
     end
 
     def append(key, value)
-      req = [REQUEST, OPCODES[:append], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[:append])
-      write(req)
-      generic_response
+      write_append_prepend :append, key, value
     end
 
     def prepend(key, value)
-      req = [REQUEST, OPCODES[:prepend], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[:prepend])
-      write(req)
-      generic_response
+      write_append_prepend :prepend, key, value
     end
 
     def stats(info='')
@@ -238,24 +380,29 @@ module Dalli
       keyvalue_response
     end
 
+    def reset_stats
+      write_generic [REQUEST, OPCODES[:stat], 'reset'.bytesize, 0, 0, 0, 'reset'.bytesize, 0, 0, 'reset'].pack(FORMAT[:stat])
+    end
+
     def cas(key)
       req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
       write(req)
-      cas_response
+      data_cas_response
     end
 
     def version
-      req = [REQUEST, OPCODES[:version], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
-      write(req)
-      generic_response
+      write_generic [REQUEST, OPCODES[:version], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
     end
 
-    COMPRESSION_MIN_SIZE = 1024
+    def touch(key, ttl)
+      ttl = sanitize_ttl(ttl)
+      write_generic [REQUEST, OPCODES[:touch], key.bytesize, 4, 0, 0, key.bytesize + 4, 0, 0, ttl, key].pack(FORMAT[:touch])
+    end
 
     # http://www.hjp.at/zettel/m/memcached_flags.rxml
     # Looks like most clients use bit 0 to indicate native language serialization
     # and bit 1 to indicate gzip compression.
-    FLAG_MARSHALLED = 0x1
+    FLAG_SERIALIZED = 0x1
     FLAG_COMPRESSED = 0x2
 
     def serialize(key, value, options=nil)
@@ -263,7 +410,9 @@ module Dalli
       value = unless options && options[:raw]
         marshalled = true
         begin
-          Marshal.dump(value)
+          self.serializer.dump(value)
+        rescue Timeout::Error => e
+          raise e
         rescue => ex
           # Marshalling can throw several different types of generic Ruby exceptions.
           # Convert to a specific exception so we can special case it higher up the stack.
@@ -275,31 +424,38 @@ module Dalli
         value.to_s
       end
       compressed = false
-      if @options[:compression] && value.bytesize >= COMPRESSION_MIN_SIZE
-        value = Zlib::Deflate.deflate(value)
+      set_compress_option = true if options && options[:compress]
+      if (@options[:compress] || set_compress_option) && value.bytesize >= @options[:compression_min_size] &&
+        (!@options[:compression_max_size] || value.bytesize <= @options[:compression_max_size])
+        value = self.compressor.compress(value)
         compressed = true
       end
-      raise Dalli::DalliError, "Value too large, memcached can only store #{@options[:value_max_bytes]} bytes per key [key: #{key}, size: #{value.bytesize}]" if value.bytesize > @options[:value_max_bytes]
+
       flags = 0
       flags |= FLAG_COMPRESSED if compressed
-      flags |= FLAG_MARSHALLED if marshalled
+      flags |= FLAG_SERIALIZED if marshalled
       [value, flags]
     end
 
     def deserialize(value, flags)
-      value = Zlib::Inflate.inflate(value) if (flags & FLAG_COMPRESSED) != 0
-      value = Marshal.load(value) if (flags & FLAG_MARSHALLED) != 0
+      value = self.compressor.decompress(value) if (flags & FLAG_COMPRESSED) != 0
+      value = self.serializer.load(value) if (flags & FLAG_SERIALIZED) != 0
       value
-    rescue TypeError, ArgumentError
-      raise DalliError, "Unable to unmarshal value: #{$!.message}"
+    rescue TypeError
+      raise if $!.message !~ /needs to have method `_load'|exception class\/object expected|instance of IO needed|incompatible marshal file format/
+      raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
+    rescue ArgumentError
+      raise if $!.message !~ /undefined class|marshal data too short/
+      raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
+    rescue NameError
+      raise if $!.message !~ /uninitialized constant/
+      raise UnmarshalError, "Unable to unmarshal value: #{$!.message}"
     rescue Zlib::Error
-      raise DalliError, "Unable to uncompress value: #{$!.message}"
+      raise UnmarshalError, "Unable to uncompress value: #{$!.message}"
     end
 
-    def cas_response
-      header = read(24)
-      raise Dalli::NetworkError, 'No response' if !header
-      (extras, type, status, count, _, cas) = header.unpack(CAS_HEADER)
+    def data_cas_response
+      (extras, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
       data = read(count) if count > 0
       if status == 1
         nil
@@ -315,15 +471,41 @@ module Dalli
 
     CAS_HEADER = '@4CCnNNQ'
     NORMAL_HEADER = '@4CCnN'
-    KV_HEADER = '@2n@6nN'
+    KV_HEADER = '@2n@6nN@16Q'
 
-    def generic_response(unpack=false)
-      header = read(24)
-      raise Dalli::NetworkError, 'No response' if !header
-      (extras, type, status, count) = header.unpack(NORMAL_HEADER)
+    def guard_max_value(key, value)
+      if value.bytesize <= @options[:value_max_bytes]
+        yield
+      else
+        message = "Value for #{key} over max size: #{@options[:value_max_bytes]} <= #{value.bytesize}"
+        raise Dalli::ValueOverMaxSize, message if @options[:error_when_over_max_size]
+
+        Dalli.logger.error "#{message} - this value may be truncated by memcached"
+        false
+      end
+    end
+
+    # https://github.com/memcached/memcached/blob/master/doc/protocol.txt#L79
+    # > An expiration time, in seconds. Can be up to 30 days. After 30 days, is treated as a unix timestamp of an exact date.
+    MAX_ACCEPTABLE_EXPIRATION_INTERVAL = 30*24*60*60 # 30 days
+    def sanitize_ttl(ttl)
+      ttl_as_i = ttl.to_i
+      return ttl_as_i if ttl_as_i <= MAX_ACCEPTABLE_EXPIRATION_INTERVAL
+      now = Time.now.to_i
+      return ttl_as_i if ttl_as_i > now # already a timestamp
+      Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached, converting to an expiration timestamp"
+      now + ttl_as_i
+    end
+
+    # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
+    class NilObject; end
+    NOT_FOUND = NilObject.new
+
+    def generic_response(unpack=false, cache_nils=false)
+      (extras, _, status, count) = read_header.unpack(NORMAL_HEADER)
       data = read(count) if count > 0
       if status == 1
-        nil
+        cache_nils ? NOT_FOUND : nil
       elsif status == 2 || status == 5
         false # Not stored, normal status for add operation
       elsif status != 0
@@ -337,12 +519,24 @@ module Dalli
       end
     end
 
+    def cas_response
+      (_, _, status, count, _, cas) = read_header.unpack(CAS_HEADER)
+      read(count) if count > 0  # this is potential data that we don't care about
+      if status == 1
+        nil
+      elsif status == 2 || status == 5
+        false # Not stored, normal status for add operation
+      elsif status != 0
+        raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
+      else
+        cas
+      end
+    end
+
     def keyvalue_response
       hash = {}
-      loop do
-        header = read(24)
-        raise Dalli::NetworkError, 'No response' if !header
-        (key_length, status, body_length) = header.unpack(KV_HEADER)
+      while true
+        (key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
         return hash if key_length == 0
         key = read(key_length)
         value = read(body_length - key_length) if body_length - key_length > 0
@@ -352,10 +546,8 @@ module Dalli
 
     def multi_response
       hash = {}
-      loop do
-        header = read(24)
-        raise Dalli::NetworkError, 'No response' if !header
-        (key_length, status, body_length) = header.unpack(KV_HEADER)
+      while true
+        (key_length, _, body_length, _) = read_header.unpack(KV_HEADER)
         return hash if key_length == 0
         flags = read(4).unpack('N')[0]
         key = read(key_length)
@@ -366,43 +558,48 @@ module Dalli
 
     def write(bytes)
       begin
-        @sock.write(bytes)
-      rescue SystemCallError, Timeout::Error
-        failure!
-        retry
+        @inprogress = true
+        result = @sock.write(bytes)
+        @inprogress = false
+        result
+      rescue SystemCallError, Timeout::Error => e
+        failure!(e)
       end
     end
 
     def read(count)
       begin
-        @sock.readfull(count)
-      rescue SystemCallError, Timeout::Error, EOFError
-        failure!
-        retry
+        @inprogress = true
+        data = @sock.readfull(count)
+        @inprogress = false
+        data
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        failure!(e)
       end
     end
 
+    def read_header
+      read(24) || raise(Dalli::NetworkError, 'No response')
+    end
+
     def connect
-      Dalli.logger.debug { "Dalli::Server#connect #{hostname}:#{port}" }
+      Dalli.logger.debug { "Dalli::Server#connect #{name}" }
 
       begin
-        if @hostname =~ /^\//
-          @sock = USocket.new(hostname)
-        elsif options[:async]
-          raise Dalli::DalliError, "EM support not enabled, as em-synchrony is not installed." if not defined?(AsyncSocket)
-          @sock = AsyncSocket.open(hostname, port, :timeout => options[:socket_timeout])
+        @pid = Process.pid
+        if socket_type == :unix
+          @sock = KSocket::UNIX.open(hostname, self, options)
         else
-          @sock = KSocket.open(hostname, port, :timeout => options[:socket_timeout])
+          @sock = KSocket::TCP.open(hostname, port, self, options)
         end
-        @version = version # trigger actual connect
         sasl_authentication if need_auth?
+        @version = version # trigger actual connect
         up!
       rescue Dalli::DalliError # SASL auth failure
         raise
-      rescue SystemCallError, Timeout::Error, EOFError, SocketError
+      rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
         # SocketError = DNS resolution failure
-        failure!
-        retry
+        failure!(e)
       end
     end
 
@@ -410,13 +607,11 @@ module Dalli
       [n >> 32, 0xFFFFFFFF & n]
     end
 
-    def longlong(a, b)
-      (a << 32) | b
-    end
-
     REQUEST = 0x80
     RESPONSE = 0x81
 
+    # Response codes taken from:
+    # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
     RESPONSE_CODES = {
       0 => 'No error',
       1 => 'Key not found',
@@ -425,9 +620,16 @@ module Dalli
       4 => 'Invalid arguments',
       5 => 'Item not stored',
       6 => 'Incr/decr on a non-numeric value',
+      7 => 'The vbucket belongs to another server',
+      8 => 'Authentication error',
+      9 => 'Authentication continue',
       0x20 => 'Authentication required',
       0x81 => 'Unknown command',
       0x82 => 'Out of memory',
+      0x83 => 'Not supported',
+      0x84 => 'Internal error',
+      0x85 => 'Busy',
+      0x86 => 'Temporary failure'
     }
 
     OPCODES = {
@@ -454,6 +656,7 @@ module Dalli
       :auth_negotiation => 0x20,
       :auth_request => 0x21,
       :auth_continue => 0x22,
+      :touch => 0x1C,
     }
 
     HEADER = "CCnCCnNNQ"
@@ -474,6 +677,7 @@ module Dalli
       :prepend => 'a*a*',
       :auth_request => 'a*a*',
       :auth_continue => 'a*a*',
+      :touch => 'Na*',
     }
     FORMAT = OP_FORMAT.inject({}) { |memo, (k, v)| memo[k] = HEADER + v; memo }
 
@@ -500,11 +704,10 @@ module Dalli
       # negotiate
       req = [REQUEST, OPCODES[:auth_negotiation], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
       write(req)
-      header = read(24)
-      raise Dalli::NetworkError, 'No response' if !header
-      (extras, type, status, count) = header.unpack(NORMAL_HEADER)
+
+      (extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
       raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
-      content = read(count)
+      content = read(count).gsub(/\u0000/, ' ')
       return (Dalli.logger.debug("Authentication not required/supported by server")) if status == 0x81
       mechanisms = content.split(' ')
       raise NotImplementedError, "Dalli only supports the PLAIN authentication mechanism" if !mechanisms.include?('PLAIN')
@@ -515,9 +718,7 @@ module Dalli
       req = [REQUEST, OPCODES[:auth_request], mechanism.bytesize, 0, 0, 0, mechanism.bytesize + msg.bytesize, 0, 0, mechanism, msg].pack(FORMAT[:auth_request])
       write(req)
 
-      header = read(24)
-      raise Dalli::NetworkError, 'No response' if !header
-      (extras, type, status, count) = header.unpack(NORMAL_HEADER)
+      (extras, _type, status, count) = read_header.unpack(NORMAL_HEADER)
       raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
       content = read(count)
       return Dalli.logger.info("Dalli/SASL: #{content}") if status == 0
@@ -527,5 +728,25 @@ module Dalli
       # (step, msg) = sasl.receive('challenge', content)
       # raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
     end
+
+    def parse_hostname(str)
+      res = str.match(/\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/)
+      raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
+      hostnam = res[2] || res[1]
+      if hostnam =~ /\A\//
+        socket_type = :unix
+        # in case of unix socket, allow only setting of weight, not port
+        raise Dalli::DalliError, "Could not parse hostname #{str}" if res[4]
+        weigh = res[3]
+      else
+        socket_type = :tcp
+        por = res[3] || DEFAULT_PORT
+        por = Integer(por)
+        weigh = res[4]
+      end
+      weigh ||= DEFAULT_WEIGHT
+      weigh = Integer(weigh)
+      return hostnam, por, weigh, socket_type
+    end
   end
 end