dalli 3.0.4 → 3.0.5

data/lib/dalli/protocol/server_config_parser.rb

@@ -8,19 +8,36 @@ module Dalli
     # socket_type.
     ##
     class ServerConfigParser
+      MEMCACHED_URI_PROTOCOL = 'memcached://'
+
       # TODO: Revisit this, especially the IP/domain part.  Likely
       # can limit character set to LDH + '.'.  Hex digit section
-      # appears to have been added to support IPv6, but as far as
-      # I can tell it doesn't work
+      # is there to support IPv6 addresses, which need to be specified with
+      # a bounding []
       SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
 
       DEFAULT_PORT = 11_211
       DEFAULT_WEIGHT = 1
 
-      def self.parse(str)
+      def self.parse(str, client_options)
+        return parse_non_uri(str, client_options) unless str.start_with?(MEMCACHED_URI_PROTOCOL)
+
+        parse_uri(str, client_options)
+      end
+
+      def self.parse_uri(str, client_options)
+        uri = URI.parse(str)
+        auth_details = {
+          username: uri.user,
+          password: uri.password
+        }
+        [uri.host, normalize_port(uri.port), DEFAULT_WEIGHT, :tcp, client_options.merge(auth_details)]
+      end
+
+      def self.parse_non_uri(str, client_options)
         res = deconstruct_string(str)
 
-        hostname = normalize_hostname(str, res)
+        hostname = normalize_host_from_match(str, res)
         if hostname.start_with?('/')
           socket_type = :unix
           port, weight = attributes_for_unix_socket(res)
@@ -28,7 +45,7 @@ module Dalli
           socket_type = :tcp
           port, weight = attributes_for_tcp_socket(res)
         end
-        [hostname, port, weight, socket_type]
+        [hostname, port, weight, socket_type, client_options]
       end
 
       def self.deconstruct_string(str)
@@ -49,7 +66,7 @@ module Dalli
         [normalize_port(res[3]), normalize_weight(res[4])]
       end
 
-      def self.normalize_hostname(str, res)
+      def self.normalize_host_from_match(str, res)
         raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
 
         res[2] || res[1]

data/lib/dalli/protocol/value_marshaller.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::ValueMarshaller compartmentalizes the logic for marshalling
+    # and unmarshalling unstructured data (values) to Memcached.  It also enforces
+    # limits on the maximum size of marshalled data.
+    ##
+    class ValueMarshaller
+      extend Forwardable
+
+      DEFAULTS = {
+        # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
+        value_max_bytes: 1024 * 1024
+      }.freeze
+
+      OPTIONS = DEFAULTS.keys.freeze
+
+      def_delegators :@value_serializer, :serializer
+      def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
+
+      def initialize(client_options)
+        @value_serializer = ValueSerializer.new(client_options)
+        @value_compressor = ValueCompressor.new(client_options)
+
+        @marshal_options =
+          DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+      end
+
+      def store(key, value, options = nil)
+        bitflags = 0
+        value, bitflags = @value_serializer.store(value, options, bitflags)
+        value, bitflags = @value_compressor.store(value, options, bitflags)
+
+        error_if_over_max_value_bytes(key, value)
+        [value, bitflags]
+      end
+
+      def retrieve(value, flags)
+        value = @value_compressor.retrieve(value, flags)
+        @value_serializer.retrieve(value, flags)
+      end
+
+      def value_max_bytes
+        @marshal_options[:value_max_bytes]
+      end
+
+      def error_if_over_max_value_bytes(key, value)
+        return if value.bytesize <= value_max_bytes
+
+        message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
+        raise Dalli::ValueOverMaxSize, message
+      end
+    end
+  end
+end

data/lib/dalli/protocol/value_serializer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::ValueSerializer compartmentalizes the logic for managing
+    # serialization and deserialization of stored values.  It manages interpreting
+    # relevant options from both client and request, determining whether to
+    # serialize/deserialize on store/retrieve, and processes bitflags as necessary.
+    ##
+    class ValueSerializer
+      DEFAULTS = {
+        serializer: Marshal
+      }.freeze
+
+      OPTIONS = DEFAULTS.keys.freeze
+
+      # https://www.hjp.at/zettel/m/memcached_flags.rxml
+      # Looks like most clients use bit 0 to indicate native language serialization
+      FLAG_SERIALIZED = 0x1
+
+      attr_accessor :serialization_options
+
+      def initialize(protocol_options)
+        @serialization_options =
+          DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
+      end
+
+      def store(value, req_options, bitflags)
+        do_serialize = !(req_options && req_options[:raw])
+        store_value = do_serialize ? serialize_value(value) : value.to_s
+        bitflags |= FLAG_SERIALIZED if do_serialize
+        [store_value, bitflags]
+      end
+
+      # TODO: Some of these error messages need to be validated.  It's not obvious
+      # that all of them are actually generated by the invoked code
+      # in current systems
+      # rubocop:disable Layout/LineLength
+      TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
+      ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
+      NAME_ERR_STR = 'uninitialized constant'
+      # rubocop:enable Layout/LineLength
+
+      def retrieve(value, bitflags)
+        serialized = (bitflags & FLAG_SERIALIZED) != 0
+        serialized ? serializer.load(value) : value
+      rescue TypeError => e
+        filter_type_error(e)
+      rescue ArgumentError => e
+        filter_argument_error(e)
+      rescue NameError => e
+        filter_name_error(e)
+      end
+
+      def filter_type_error(err)
+        raise err unless TYPE_ERR_REGEXP.match?(err.message)
+
+        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+      end
+
+      def filter_argument_error(err)
+        raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
+
+        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+      end
+
+      def filter_name_error(err)
+        raise err unless err.message.include?(NAME_ERR_STR)
+
+        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+      end
+
+      def serializer
+        @serialization_options[:serializer]
+      end
+
+      def serialize_value(value)
+        serializer.dump(value)
+      rescue Timeout::Error => e
+        raise e
+      rescue StandardError => e
+        # Serializing can throw several different types of generic Ruby exceptions.
+        # Convert to a specific exception so we can special case it higher up the stack.
+        exc = Dalli::MarshalError.new(e.message)
+        exc.set_backtrace e.backtrace
+        raise exc
+      end
+    end
+  end
+end

data/lib/dalli/protocol.rb

@@ -2,8 +2,7 @@
 
 module Dalli
   module Protocol
-    # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
-    class NilObject; end
-    NOT_FOUND = NilObject.new
+    # Preserved for backwards compatibility.  Should be removed in 4.0
+    NOT_FOUND = ::Dalli::NOT_FOUND
   end
 end

data/lib/dalli/ring.rb

@@ -1,10 +1,24 @@
 # frozen_string_literal: true
 
-require "digest/sha1"
-require "zlib"
+require 'digest/sha1'
+require 'zlib'
 
 module Dalli
+  ##
+  # An implementation of a consistent hash ring, designed to minimize
+  # the cache miss impact of adding or removing servers from the ring.
+  # That is, adding or removing a server from the ring should impact
+  # the key -> server mapping of ~ 1/N of the stored keys where N is the
+  # number of servers in the ring.  This is done by creating a large
+  # number of "points" per server, distributed over the space
+  # 0x00000000 - 0xFFFFFFFF. For a given key, we calculate the CRC32
+  # hash, and find the nearest "point" that is less than or equal to the
+  # the key's hash.  In this implemetation, each "point" is represented
+  # by a Dalli::Ring::Entry.
+  ##
   class Ring
+    # The number of entries on the continuum created per server
+    # in an equally weighted scenario.
     POINTS_PER_SERVER = 160 # this is the default in libmemcached
 
     attr_accessor :servers, :continuum
@@ -12,45 +26,48 @@ module Dalli
     def initialize(servers, options)
       @servers = servers
       @continuum = nil
-      if servers.size > 1
-        total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
-        continuum = []
-        servers.each do |server|
-          entry_count_for(server, servers.size, total_weight).times do |idx|
-            hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
-            value = Integer("0x#{hash[0..7]}")
-            continuum << Dalli::Ring::Entry.new(value, server)
-          end
-        end
-        @continuum = continuum.sort_by(&:value)
-      end
+      @continuum = build_continuum(servers) if servers.size > 1
 
       threadsafe! unless options[:threadsafe] == false
       @failover = options[:failover] != false
     end
 
     def server_for_key(key)
-      if @continuum
-        hkey = hash_for(key)
-        20.times do |try|
-          # Find the closest index in the Ring with value <= the given value
-          entryidx = @continuum.bsearch_index { |entry| entry.value > hkey }
-          if entryidx.nil?
-            entryidx = @continuum.size - 1
-          else
-            entryidx -= 1
-          end
-          server = @continuum[entryidx].server
-          return server if server.alive?
-          break unless @failover
-          hkey = hash_for("#{try}#{key}")
-        end
-      else
-        server = @servers.first
-        return server if server&.alive?
+      server = if @continuum
+                 server_from_continuum(key)
+               else
+                 @servers.first
+               end
+
+      # Note that the call to alive? has the side effect of initializing
+      # the socket
+      return server if server&.alive?
+
+      raise Dalli::RingError, 'No server available'
+    end
+
+    def server_from_continuum(key)
+      hkey = hash_for(key)
+      20.times do |try|
+        server = server_for_hash_key(hkey)
+
+        # Note that the call to alive? has the side effect of initializing
+        # the socket
+        return server if server.alive?
+        break unless @failover
+
+        hkey = hash_for("#{try}#{key}")
       end
+      nil
+    end
 
-      raise Dalli::RingError, "No server available"
+    def keys_grouped_by_server(key_arr)
+      key_arr.group_by do |key|
+        server_for_key(key)
+      rescue Dalli::RingError
+        Dalli.logger.debug { "unable to get key #{key}" }
+        nil
+      end
     end
 
     def lock
@@ -62,6 +79,19 @@ module Dalli
       end
     end
 
+    def flush_multi_responses
+      @servers.each do |s|
+        s.request(:noop)
+      rescue Dalli::NetworkError
+        # Ignore this error, as it indicates the socket is unavailable
+        # and there's no need to flush
+      end
+    end
+
+    def socket_timeout
+      @servers.first.socket_timeout
+    end
+
     private
 
     def threadsafe!
@@ -78,9 +108,35 @@ module Dalli
       ((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
     end
 
+    def server_for_hash_key(hash_key)
+      # Find the closest index in the Ring with value <= the given value
+      entryidx = @continuum.bsearch_index { |entry| entry.value > hash_key }
+      if entryidx.nil?
+        entryidx = @continuum.size - 1
+      else
+        entryidx -= 1
+      end
+      @continuum[entryidx].server
+    end
+
+    def build_continuum(servers)
+      continuum = []
+      total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
+      servers.each do |server|
+        entry_count_for(server, servers.size, total_weight).times do |idx|
+          hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
+          value = Integer("0x#{hash[0..7]}")
+          continuum << Dalli::Ring::Entry.new(value, server)
+        end
+      end
+      continuum.sort_by(&:value)
+    end
+
+    ##
+    # Represents a point in the consistent hash ring implementation.
+    ##
     class Entry
-      attr_reader :value
-      attr_reader :server
+      attr_reader :value, :server
 
       def initialize(val, srv)
         @value = val

data/lib/dalli/server.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module Dalli
-  warn "Dalli::Server is deprecated, use Dalli::Protocol::Binary instead"
+module Dalli # rubocop:disable Style/Documentation
+  warn 'Dalli::Server is deprecated, use Dalli::Protocol::Binary instead'
   Server = Protocol::Binary
 end

data/lib/dalli/servers_arg_normalizer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Dalli
+  ##
+  # This module contains methods for validating and normalizing the servers
+  # argument passed to the client.  This argument can be nil, a string, or
+  # an array of strings.  Each string value in the argument can represent
+  # a single server or a comma separated list of servers.
+  #
+  # If nil, it falls back to the values of ENV['MEMCACHE_SERVERS'] if the latter is
+  # defined.  If that environment value is not defined, a default of '127.0.0.1:11211'
+  # is used.
+  #
+  # A server config string can take one of three forms:
+  #   * A colon separated string of (host, port, weight) where both port and
+  #     weight are optional (e.g. 'localhost', 'abc.com:12345', 'example.org:22222:3')
+  #   * A colon separated string of (UNIX socket, weight) where the weight is optional
+  #     (e.g. '/var/run/memcached/socket', '/tmp/xyz:3') (not supported on Windows)
+  #   * A URI with a 'memcached' protocol, which will typically include a username/password
+  #
+  # The methods in this module do not validate the format of individual server strings, but
+  # rather normalize the argument into a compact array, wherein each array entry corresponds
+  # to a single server config string.  If that normalization is not possible, then an
+  # ArgumentError is thrown.
+  ##
+  module ServersArgNormalizer
+    ENV_VAR_NAME = 'MEMCACHE_SERVERS'
+    DEFAULT_SERVERS = ['127.0.0.1:11211'].freeze
+
+    ##
+    # Normalizes the argument into an array of servers.
+    # If the argument is a string, or an array containing strings, it's expected that the URIs are comma separated e.g.
+    # "memcache1.example.com:11211,memcache2.example.com:11211,memcache3.example.com:11211"
+    def self.normalize_servers(arg)
+      arg = apply_defaults(arg)
+      validate_type(arg)
+      Array(arg).flat_map { |s| s.split(',') }.reject(&:empty?)
+    end
+
+    def self.apply_defaults(arg)
+      return arg unless arg.nil?
+
+      ENV[ENV_VAR_NAME] || DEFAULT_SERVERS
+    end
+
+    def self.validate_type(arg)
+      return if arg.is_a?(String)
+      return if arg.is_a?(Array) && arg.all?(String)
+
+      raise ArgumentError,
+            'An explicit servers argument must be a comma separated string or an array containing strings.'
+    end
+  end
+end

data/lib/dalli/socket.rb

@@ -4,56 +4,85 @@ require 'openssl'
 require 'rbconfig'
 
 module Dalli
+  ##
+  # Various socket implementations used by Dalli.
+  ##
   module Socket
+    ##
+    # Common methods for all socket implementations.
+    ##
     module InstanceMethods
-
       def readfull(count)
-        value = +""
+        value = +''
         loop do
           result = read_nonblock(count - value.bytesize, exception: false)
-          if result == :wait_readable
-            raise Timeout::Error, "IO timeout: #{safe_options.inspect}" unless IO.select([self], nil, nil, options[:socket_timeout])
-          elsif result == :wait_writable
-            raise Timeout::Error, "IO timeout: #{safe_options.inspect}" unless IO.select(nil, [self], nil, options[:socket_timeout])
-          elsif result
-            value << result
-          else
-            raise Errno::ECONNRESET, "Connection reset: #{safe_options.inspect}"
-          end
+          value << result if append_to_buffer?(result)
           break if value.bytesize == count
         end
         value
       end
 
       def read_available
-        value = +""
+        value = +''
         loop do
           result = read_nonblock(8196, exception: false)
-          if result == :wait_readable
-            break
-          elsif result == :wait_writable
-            break
-          elsif result
-            value << result
-          else
-            raise Errno::ECONNRESET, "Connection reset: #{safe_options.inspect}"
-          end
+          break if WAIT_RCS.include?(result)
+          raise Errno::ECONNRESET, "Connection reset: #{logged_options.inspect}" unless result
+
+          value << result
         end
         value
       end
 
-      def safe_options
-        options.reject { |k, v| [:username, :password].include? k }
+      WAIT_RCS = %i[wait_writable wait_readable].freeze
+
+      def append_to_buffer?(result)
+        raise Timeout::Error, "IO timeout: #{logged_options.inspect}" if nonblock_timed_out?(result)
+        raise Errno::ECONNRESET, "Connection reset: #{logged_options.inspect}" unless result
+
+        !WAIT_RCS.include?(result)
+      end
+
+      def nonblock_timed_out?(result)
+        return true if result == :wait_readable && !wait_readable(options[:socket_timeout])
+
+        # TODO: Do we actually need this?  Looks to be only used in read_nonblock
+        result == :wait_writable && !wait_writable(options[:socket_timeout])
+      end
+
+      FILTERED_OUT_OPTIONS = %i[username password].freeze
+      def logged_options
+        options.reject { |k, _| FILTERED_OUT_OPTIONS.include? k }
       end
     end
 
+    ##
+    # Wraps the below TCP socket class in the case where the client
+    # has configured a TLS/SSL connection between Dalli and the
+    # Memcached server.
+    ##
     class SSLSocket < ::OpenSSL::SSL::SSLSocket
       include Dalli::Socket::InstanceMethods
       def options
         io.options
       end
+
+      unless method_defined?(:wait_readable)
+        def wait_readable(timeout = nil)
+          to_io.wait_readable(timeout)
+        end
+      end
+
+      unless method_defined?(:wait_writable)
+        def wait_writable(timeout = nil)
+          to_io.wait_writable(timeout)
+        end
+      end
     end
 
+    ##
+    # A standard TCP socket between the Dalli client and the Memcached server.
+    ##
     class TCP < TCPSocket
       include Dalli::Socket::InstanceMethods
       attr_accessor :options, :server
@@ -61,42 +90,57 @@ module Dalli
       def self.open(host, port, server, options = {})
         Timeout.timeout(options[:socket_timeout]) do
           sock = new(host, port)
-          sock.options = {host: host, port: port}.merge(options)
+          sock.options = { host: host, port: port }.merge(options)
           sock.server = server
-          sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, true)
-          sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_KEEPALIVE, true) if options[:keepalive]
-          sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_RCVBUF, options[:rcvbuf]) if options[:rcvbuf]
-          sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_SNDBUF, options[:sndbuf]) if options[:sndbuf]
-
-          return sock unless options[:ssl_context]
-
-          ssl_socket = Dalli::Socket::SSLSocket.new(sock, options[:ssl_context])
-          ssl_socket.hostname = host
-          ssl_socket.sync_close = true
-          ssl_socket.connect
-          ssl_socket
+          init_socket_options(sock, options)
+
+          options[:ssl_context] ? wrapping_ssl_socket(sock, host, options[:ssl_context]) : sock
         end
       end
-    end
-  end
 
-  if RbConfig::CONFIG['host_os'] =~ /mingw|mswin/
-    class Dalli::Socket::UNIX
-      def initialize(*args)
-        raise Dalli::DalliError, 'Unix sockets are not supported on Windows platform.'
+      def self.init_socket_options(sock, options)
+        sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, true)
+        sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_KEEPALIVE, true) if options[:keepalive]
+        sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_RCVBUF, options[:rcvbuf]) if options[:rcvbuf]
+        sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_SNDBUF, options[:sndbuf]) if options[:sndbuf]
+      end
+
+      def self.wrapping_ssl_socket(tcp_socket, host, ssl_context)
+        ssl_socket = Dalli::Socket::SSLSocket.new(tcp_socket, ssl_context)
+        ssl_socket.hostname = host
+        ssl_socket.sync_close = true
+        ssl_socket.connect
+        ssl_socket
       end
     end
-  else
-    class Dalli::Socket::UNIX < UNIXSocket
-      include Dalli::Socket::InstanceMethods
-      attr_accessor :options, :server
 
-      def self.open(path, server, options = {})
-        Timeout.timeout(options[:socket_timeout]) do
-          sock = new(path)
-          sock.options = {path: path}.merge(options)
-          sock.server = server
-          sock
+    if /mingw|mswin/.match?(RbConfig::CONFIG['host_os'])
+      ##
+      # UNIX domain sockets are not supported on Windows platforms.
+      ##
+      class UNIX
+        def initialize(*_args)
+          raise Dalli::DalliError, 'Unix sockets are not supported on Windows platform.'
+        end
+      end
+    else
+
+      ##
+      # UNIX represents a UNIX domain socket, which is an interprocess communication
+      # mechanism between processes on the same host.  Used when the Memcached server
+      # is running on the same machine as the Dalli client.
+      ##
+      class UNIX < UNIXSocket
+        include Dalli::Socket::InstanceMethods
+        attr_accessor :options, :server
+
+        def self.open(path, server, options = {})
+          Timeout.timeout(options[:socket_timeout]) do
+            sock = new(path)
+            sock.options = { path: path }.merge(options)
+            sock.server = server
+            sock
+          end
         end
       end
     end

data/lib/dalli/version.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = "3.0.4"
+  VERSION = '3.0.5'
+
+  MIN_SUPPORTED_MEMCACHED_VERSION = '1.4'
 end