dalli 3.0.1 → 3.0.5

data/lib/dalli/client.rb

@@ -1,17 +1,24 @@
 # frozen_string_literal: true
 
-require "digest/md5"
-require "set"
+require 'digest/md5'
+require 'set'
 
 # encoding: ascii
 module Dalli
+  ##
+  # Dalli::Client is the main class which developers will use to interact with
+  # Memcached.
+  ##
   class Client
     ##
     # Dalli::Client is the main class which developers will use to interact with
     # the memcached server.  Usage:
     #
-    #   Dalli::Client.new(['localhost:11211:10', 'cache-2.example.com:11211:5', '192.168.0.1:22122:5', '/var/run/memcached/socket'],
-    #                   :threadsafe => true, :failover => true, :expires_in => 300)
+    #   Dalli::Client.new(['localhost:11211:10',
+    #                      'cache-2.example.com:11211:5',
+    #                      '192.168.0.1:22122:5',
+    #                      '/var/run/memcached/socket'],
+    #                     failover: true, expires_in: 300)
     #
     # servers is an Array of "host:port:weight" where weight allows you to distribute cache unevenly.
     # Both weight and port are optional.  If you pass in nil, Dalli will use the <tt>MEMCACHE_SERVERS</tt>
@@ -24,18 +31,25 @@ module Dalli
     # - :namespace - prepend each key with this value to provide simple namespacing.
     # - :failover - if a server is down, look for and store values on another server in the ring.  Default: true.
     # - :threadsafe - ensure that only one thread is actively using a socket at a time. Default: true.
-    # - :expires_in - default TTL in seconds if you do not pass TTL as a parameter to an individual operation, defaults to 0 or forever
-    # - :compress - defaults to false, if true Dalli will compress values larger than 1024 bytes before sending them to memcached.
+    # - :expires_in - default TTL in seconds if you do not pass TTL as a parameter to an individual operation, defaults
+    #                 to 0 or forever.
+    # - :compress - if true Dalli will compress values larger than compression_min_size bytes before sending them
+    #               to memcached.  Default: true.
+    # - :compression_min_size - the minimum size (in bytes) for which Dalli will compress values sent to Memcached.
+    #                           Defaults to 4K.
     # - :serializer - defaults to Marshal
-    # - :compressor - defaults to zlib
-    # - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for #fetch operations.
-    # - :digest_class - defaults to Digest::MD5, allows you to pass in an object that responds to the hexdigest method, useful for injecting a FIPS compliant hash object.
-    # - :protocol_implementation - defaults to Dalli::Protocol::Binary which uses the binary protocol. Allows you to pass an alternative implementation using another protocol.
+    # - :compressor - defaults to Dalli::Compressor, a Zlib-based implementation
+    # - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for
+    #                 #fetch operations.
+    # - :digest_class - defaults to Digest::MD5, allows you to pass in an object that responds to the hexdigest method,
+    #                   useful for injecting a FIPS compliant hash object.
+    # - :protocol_implementation - defaults to Dalli::Protocol::Binary which uses the binary protocol. Allows you to
+    #                              pass an alternative implementation using another protocol.
     #
     def initialize(servers = nil, options = {})
-      validate_servers_arg(servers)
-      @servers = normalize_servers(servers || ENV["MEMCACHE_SERVERS"] || "127.0.0.1:11211")
+      @servers = ::Dalli::ServersArgNormalizer.normalize_servers(servers)
       @options = normalize_options(options)
+      @key_manager = ::Dalli::KeyManager.new(options)
       @ring = nil
     end
 
@@ -49,9 +63,11 @@ module Dalli
     # pipelined as Dalli will use 'quiet' operations where possible.
     # Currently supports the set, add, replace and delete operations.
     def multi
-      old, Thread.current[:dalli_multi] = Thread.current[:dalli_multi], true
+      old = Thread.current[:dalli_multi]
+      Thread.current[:dalli_multi] = true
       yield
     ensure
+      @ring&.flush_multi_responses
       Thread.current[:dalli_multi] = old
     end
 
@@ -71,6 +87,7 @@ module Dalli
       keys.compact!
 
       return {} if keys.empty?
+
       if block_given?
         get_multi_yielder(keys) { |k, data| yield k, data.first }
       else
@@ -80,7 +97,7 @@ module Dalli
       end
     end
 
-    CACHE_NILS = {cache_nils: true}.freeze
+    CACHE_NILS = { cache_nils: true }.freeze
 
     # Fetch the value associated with the key.
     # If a value is found, then it is returned.
@@ -90,19 +107,24 @@ module Dalli
     # If a value is not found (or if the found value is nil and :cache_nils is false)
     # and a block is given, the block will be invoked and its return value
     # written to the cache and returned.
-    def fetch(key, ttl = nil, options = nil)
-      options = options.nil? ? CACHE_NILS : options.merge(CACHE_NILS) if @options[:cache_nils]
-      val = get(key, options)
-      not_found = @options[:cache_nils] ?
-        val == Dalli::Protocol::NOT_FOUND :
-        val.nil?
-      if not_found && block_given?
+    def fetch(key, ttl = nil, req_options = nil)
+      req_options = req_options.nil? ? CACHE_NILS : req_options.merge(CACHE_NILS) if cache_nils
+      val = get(key, req_options)
+      if not_found?(val) && block_given?
         val = yield
-        add(key, val, ttl_or_default(ttl), options)
+        add(key, val, ttl_or_default(ttl), req_options)
       end
       val
     end
 
+    def not_found?(val)
+      cache_nils ? val == ::Dalli::NOT_FOUND : val.nil?
+    end
+
+    def cache_nils
+      @options[:cache_nils]
+    end
+
     ##
     # compare and swap values using optimistic locking.
     # Fetch the existing value for key.
@@ -170,7 +192,7 @@ module Dalli
       ring.servers.map { |s| s.request(:flush, time += delay) }
     end
 
-    alias_method :flush_all, :flush
+    alias flush_all flush
 
     ##
     # Incr adds the given amount to the counter on the memcached server.
@@ -184,7 +206,8 @@ module Dalli
     # exist.  To increase an existing counter and update its TTL, use
     # #cas.
     def incr(key, amt = 1, ttl = nil, default = nil)
-      raise ArgumentError, "Positive values only: #{amt}" if amt < 0
+      raise ArgumentError, "Positive values only: #{amt}" if amt.negative?
+
       perform(:incr, key, amt.to_i, ttl_or_default(ttl), default)
     end
 
@@ -203,7 +226,8 @@ module Dalli
     # exist.  To decrease an existing counter and update its TTL, use
     # #cas.
     def decr(key, amt = 1, ttl = nil, default = nil)
-      raise ArgumentError, "Positive values only: #{amt}" if amt < 0
+      raise ArgumentError, "Positive values only: #{amt}" if amt.negative?
+
       perform(:decr, key, amt.to_i, ttl_or_default(ttl), default)
     end
 
@@ -248,7 +272,7 @@ module Dalli
     ##
     ## Make sure memcache servers are alive, or raise an Dalli::RingError
     def alive!
-      ring.server_for_key("")
+      ring.server_for_key('')
     end
 
     ##
@@ -266,7 +290,7 @@ module Dalli
     # value and CAS will be passed to the block.
     def get_cas(key)
       (value, cas) = perform(:cas, key)
-      value = !value || value == "Not found" ? nil : value
+      value = nil if !value || value == 'Not found'
       if block_given?
         yield value, cas
       else
@@ -317,12 +341,12 @@ module Dalli
     # Close our connection to each server.
     # If you perform another operation after this, the connections will be re-established.
     def close
-      if @ring
-        @ring.servers.each { |s| s.close }
-        @ring = nil
-      end
+      return unless @ring
+
+      @ring.servers.each(&:close)
+      @ring = nil
     end
-    alias_method :reset, :close
+    alias reset close
 
     # Stub method so a bare Dalli client can pretend to be a connection pool.
     def with
@@ -333,8 +357,9 @@ module Dalli
 
     def cas_core(key, always_set, ttl = nil, options = nil)
       (value, cas) = perform(:cas, key)
-      value = !value || value == "Not found" ? nil : value
+      value = nil if !value || value == 'Not found'
       return if value.nil? && !always_set
+
       newvalue = yield(value)
       perform(:set, key, newvalue, ttl_or_default(ttl), cas, options)
     end
@@ -345,26 +370,90 @@ module Dalli
       raise ArgumentError, "Cannot convert ttl (#{ttl}) to an integer"
     end
 
-    def groups_for_keys(*keys)
-      keys.flatten!
-      keys.map! { |a| validate_key(a.to_s) }
+    def ring
+      # TODO: This server initialization should probably be pushed down
+      # to the Ring
+      @ring ||= Dalli::Ring.new(
+        @servers.map do |s|
+          protocol_implementation.new(s, @options)
+        end, @options
+      )
+    end
 
-      keys.group_by { |key|
-        begin
-          ring.server_for_key(key)
-        rescue Dalli::RingError
-          Dalli.logger.debug { "unable to get key #{key}" }
-          nil
+    def protocol_implementation
+      @protocol_implementation ||= @options.fetch(:protocol_implementation, Dalli::Protocol::Binary)
+    end
+
+    # Chokepoint method for instrumentation
+    def perform(*all_args)
+      return yield if block_given?
+
+      op, key, *args = all_args
+
+      key = key.to_s
+      key = @key_manager.validate_key(key)
+
+      server = ring.server_for_key(key)
+      server.request(op, key, *args)
+    rescue NetworkError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { 'retrying request with new server' }
+      retry
+    end
+
+    def normalize_options(opts)
+      begin
+        opts[:expires_in] = opts[:expires_in].to_i if opts[:expires_in]
+      rescue NoMethodError
+        raise ArgumentError, "cannot convert :expires_in => #{opts[:expires_in].inspect} to an integer"
+      end
+      opts
+    end
+
+    # TODO: Look at extracting below into separate MultiYielder class
+
+    ##
+    # Yields, one at a time, keys and their values+attributes.
+    #
+    def get_multi_yielder(keys, &block)
+      return {} if keys.empty?
+
+      ring.lock do
+        groups = groups_for_keys(keys)
+        if (unfound_keys = groups.delete(nil))
+          Dalli.logger.debug do
+            "unable to get keys for #{unfound_keys.length} keys "\
+              'because no matching server was found'
+          end
+        end
+        make_multi_get_requests(groups)
+
+        servers = groups.keys
+        return if servers.empty?
+
+        # TODO: How does this exit on a NetworkError
+        servers = perform_multi_response_start(servers)
+
+        timeout = servers.first.options[:socket_timeout]
+        start_time = Time.now
+        loop do
+          # remove any dead servers
+          # TODO: Is this well behaved in a multi-threaded environment?
+          # Accessing the server socket like this seems problematic
+          servers.delete_if { |s| s.sock.nil? }
+          break if servers.empty?
+
+          servers = multi_yielder_loop(servers, start_time, timeout, &block)
         end
-      }
+      end
+    rescue NetworkError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { 'retrying multi yielder because of timeout' }
+      retry
     end
 
     def make_multi_get_requests(groups)
       groups.each do |server, keys_for_server|
-        # TODO: do this with the perform chokepoint?
-        # But given the fact that fetching the response doesn't take place
-        # in that slot it's misleading anyway. Need to move all of this method
-        # into perform to be meaningful
         server.request(:send_multiget, keys_for_server)
       rescue DalliError, NetworkError => e
         Dalli.logger.debug { e.inspect }
@@ -372,175 +461,86 @@ module Dalli
       end
     end
 
+    # raises Dalli::NetworkError
     def perform_multi_response_start(servers)
+      deleted = []
+
       servers.each do |server|
         next unless server.alive?
+
         begin
           server.multi_response_start
-        rescue DalliError, NetworkError => e
+        rescue Dalli::NetworkError
+          abort_multi_response(servers)
+          raise
+        rescue Dalli::DalliError => e
           Dalli.logger.debug { e.inspect }
-          Dalli.logger.debug { "results from this server will be missing" }
-          servers.delete(server)
+          Dalli.logger.debug { 'results from this server will be missing' }
+          deleted.append(server)
         end
       end
-      servers
-    end
 
-    ##
-    # Ensures that the servers arg is either an array or a string.
-    def validate_servers_arg(servers)
-      return if servers.nil?
-      return if servers.is_a?(Array)
-      return if servers.is_a?(String)
+      servers.delete_if { |server| deleted.include?(server) }
+    end
 
-      raise ArgumentError, "An explicit servers argument must be a comma separated string or an array containing strings."
+    # Swallows Dalli::NetworkError
+    def abort_multi_response(servers)
+      servers.each(&:multi_response_abort)
     end
 
-    ##
-    # Normalizes the argument into an array of servers.
-    # If the argument is a string, or an array containing strings, it's expected that the URIs are comma separated e.g.
-    # "memcache1.example.com:11211,memcache2.example.com:11211,memcache3.example.com:11211"
-    def normalize_servers(servers)
-      Array(servers).flat_map do |server|
-        if server.is_a? String
-          server.split(",")
-        else
-          server
-        end
+    def multi_yielder_loop(servers, start_time, timeout, &block)
+      time_left = remaining_time(start_time, timeout)
+      readable_servers = servers_with_data(servers, time_left)
+      if readable_servers.empty?
+        abort_multi_connections_w_timeout(servers)
+        return readable_servers
       end
-    end
 
-    def ring
-      @ring ||= Dalli::Ring.new(
-        @servers.map { |s|
-          server_options = {}
-          if s.start_with?("memcached://")
-            uri = URI.parse(s)
-            server_options[:username] = uri.user
-            server_options[:password] = uri.password
-            s = "#{uri.host}:#{uri.port}"
-          end
-          @options.fetch(:protocol_implementation, Dalli::Protocol::Binary).new(s, @options.merge(server_options))
-        }, @options
-      )
+      readable_servers.each do |server|
+        servers.delete(server) if respond_to_readable_server(server, &block)
+      end
+      servers
+    rescue NetworkError
+      abort_multi_response(servers)
+      raise
     end
 
-    # Chokepoint method for instrumentation
-    def perform(*all_args)
-      return yield if block_given?
-      op, key, *args = all_args
+    def remaining_time(start, timeout)
+      elapsed = Time.now - start
+      return 0 if elapsed > timeout
 
-      key = key.to_s
-      key = validate_key(key)
-      begin
-        server = ring.server_for_key(key)
-        server.request(op, key, *args)
-      rescue NetworkError => e
-        Dalli.logger.debug { e.inspect }
-        Dalli.logger.debug { "retrying request with new server" }
-        retry
-      end
+      timeout - elapsed
     end
 
-    def validate_key(key)
-      raise ArgumentError, "key cannot be blank" if !key || key.length == 0
-      key = key_with_namespace(key)
-      if key.length > 250
-        digest_class = @options[:digest_class] || ::Digest::MD5
-        max_length_before_namespace = 212 - (namespace || "").size
-        key = "#{key[0, max_length_before_namespace]}:md5:#{digest_class.hexdigest(key)}"
+    # Swallows Dalli::NetworkError
+    def abort_multi_connections_w_timeout(servers)
+      abort_multi_response(servers)
+      servers.each do |server|
+        Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
       end
-      key
-    end
 
-    def key_with_namespace(key)
-      (ns = namespace) ? "#{ns}:#{key}" : key
+      true # Required to simplify caller
     end
 
-    def key_without_namespace(key)
-      (ns = namespace) ? key.sub(%r{\A#{Regexp.escape ns}:}, "") : key
-    end
+    def respond_to_readable_server(server)
+      server.multi_response_nonblock.each_pair do |key, value_list|
+        yield @key_manager.key_without_namespace(key), value_list
+      end
 
-    def namespace
-      return nil unless @options[:namespace]
-      @options[:namespace].is_a?(Proc) ? @options[:namespace].call.to_s : @options[:namespace].to_s
+      server.multi_response_completed?
     end
 
-    def normalize_options(opts)
-      if opts[:compression]
-        Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just :compress => true.  Please update your configuration."
-        opts[:compress] = opts.delete(:compression)
-      end
-      begin
-        opts[:expires_in] = opts[:expires_in].to_i if opts[:expires_in]
-      rescue NoMethodError
-        raise ArgumentError, "cannot convert :expires_in => #{opts[:expires_in].inspect} to an integer"
-      end
-      if opts[:digest_class] && !opts[:digest_class].respond_to?(:hexdigest)
-        raise ArgumentError, "The digest_class object must respond to the hexdigest method"
-      end
-      opts
-    end
+    def servers_with_data(servers, timeout)
+      readable, = IO.select(servers.map(&:sock), nil, nil, timeout)
+      return [] if readable.nil?
 
-    ##
-    # Yields, one at a time, keys and their values+attributes.
-    def get_multi_yielder(keys)
-      perform do
-        return {} if keys.empty?
-        ring.lock do
-          groups = groups_for_keys(keys)
-          if (unfound_keys = groups.delete(nil))
-            Dalli.logger.debug { "unable to get keys for #{unfound_keys.length} keys because no matching server was found" }
-          end
-          make_multi_get_requests(groups)
-
-          servers = groups.keys
-          return if servers.empty?
-          servers = perform_multi_response_start(servers)
-
-          start = Time.now
-          loop do
-            # remove any dead servers
-            servers.delete_if { |s| s.sock.nil? }
-            break if servers.empty?
-
-            # calculate remaining timeout
-            elapsed = Time.now - start
-            timeout = servers.first.options[:socket_timeout]
-            time_left = elapsed > timeout ? 0 : timeout - elapsed
-
-            sockets = servers.map(&:sock)
-            readable, _ = IO.select(sockets, nil, nil, time_left)
-
-            if readable.nil?
-              # no response within timeout; abort pending connections
-              servers.each do |server|
-                Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
-                server.multi_response_abort
-              end
-              break
-
-            else
-              readable.each do |sock|
-                server = sock.server
-
-                begin
-                  server.multi_response_nonblock.each_pair do |key, value_list|
-                    yield key_without_namespace(key), value_list
-                  end
-
-                  if server.multi_response_completed?
-                    servers.delete(server)
-                  end
-                rescue NetworkError
-                  servers.delete(server)
-                end
-              end
-            end
-          end
-        end
-      end
+      readable.map(&:server)
     end
 
+    def groups_for_keys(*keys)
+      keys.flatten!
+      keys.map! { |a| @key_manager.validate_key(a.to_s) }
+      ring.keys_grouped_by_server(keys)
+    end
   end
 end

data/lib/dalli/compressor.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
 
-require "zlib"
-require "stringio"
+require 'zlib'
+require 'stringio'
 
 module Dalli
+  ##
+  # Default compressor used by Dalli, that uses
+  # Zlib DEFLATE to compress data.
+  ##
   class Compressor
     def self.compress(data)
       Zlib::Deflate.deflate(data)
@@ -14,9 +18,14 @@ module Dalli
     end
   end
 
+  ##
+  # Alternate compressor for Dalli, that uses
+  # Gzip.  Gzip adds a checksum to each compressed
+  # entry.
+  ##
   class GzipCompressor
     def self.compress(data)
-      io = StringIO.new(+"", "w")
+      io = StringIO.new(+'', 'w')
       gz = Zlib::GzipWriter.new(io)
       gz.write(data)
       gz.close
@@ -24,7 +33,7 @@ module Dalli
     end
 
     def self.decompress(data)
-      io = StringIO.new(data, "rb")
+      io = StringIO.new(data, 'rb')
       Zlib::GzipReader.new(io).read
     end
   end

data/lib/dalli/key_manager.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'digest/md5'
+
+module Dalli
+  ##
+  # This class manages and validates keys sent to Memcached, ensuring
+  # that they meet Memcached key length requirements, and supporting
+  # the implementation of optional namespaces on a per-Dalli client
+  # basis.
+  ##
+  class KeyManager
+    MAX_KEY_LENGTH = 250
+
+    NAMESPACE_SEPARATOR = ':'
+
+    # This is a hard coded md5 for historical reasons
+    TRUNCATED_KEY_SEPARATOR = ':md5:'
+
+    # This is 249 for historical reasons
+    TRUNCATED_KEY_TARGET_SIZE = 249
+
+    DEFAULTS = {
+      digest_class: ::Digest::MD5
+    }.freeze
+
+    OPTIONS = %i[digest_class namespace].freeze
+
+    attr_reader :namespace
+
+    def initialize(client_options)
+      @key_options =
+        DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+      validate_digest_class_option(@key_options)
+
+      @namespace = namespace_from_options
+    end
+
+    ##
+    # Validates the key, and transforms as needed.
+    #
+    # If the key is nil or empty, raises ArgumentError.  Whitespace
+    # characters are allowed for historical reasons, but likely shouldn't
+    # be used.
+    # If the key (with namespace) is shorter than the memcached maximum
+    # allowed key length, just returns the argument key
+    # Otherwise computes a "truncated" key that uses a truncated prefix
+    # combined with a 32-byte hex digest of the whole key.
+    ##
+    def validate_key(key)
+      raise ArgumentError, 'key cannot be blank' unless key&.length&.positive?
+
+      key = key_with_namespace(key)
+      key.length > MAX_KEY_LENGTH ? truncated_key(key) : key
+    end
+
+    ##
+    # Returns the key with the namespace prefixed, if a namespace is
+    # defined.  Otherwise just returns the key
+    ##
+    def key_with_namespace(key)
+      return key if namespace.nil?
+
+      "#{namespace}#{NAMESPACE_SEPARATOR}#{key}"
+    end
+
+    def key_without_namespace(key)
+      return key if namespace.nil?
+
+      key.sub(namespace_regexp, '')
+    end
+
+    def digest_class
+      @digest_class ||= @key_options[:digest_class]
+    end
+
+    def namespace_regexp
+      @namespace_regexp ||= /\A#{Regexp.escape(namespace)}:/.freeze unless namespace.nil?
+    end
+
+    def validate_digest_class_option(opts)
+      return if opts[:digest_class].respond_to?(:hexdigest)
+
+      raise ArgumentError, 'The digest_class object must respond to the hexdigest method'
+    end
+
+    def namespace_from_options
+      raw_namespace = @key_options[:namespace]
+      return nil unless raw_namespace
+      return raw_namespace.call.to_s if raw_namespace.is_a?(Proc)
+
+      raw_namespace.to_s
+    end
+
+    ##
+    # Produces a truncated key, if the raw key is longer than the maximum allowed
+    # length.  The truncated key is produced by generating a hex digest
+    # of the key, and appending that to a truncated section of the key.
+    ##
+    def truncated_key(key)
+      digest = digest_class.hexdigest(key)
+      "#{key[0, prefix_length(digest)]}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
+    end
+
+    def prefix_length(digest)
+      return TRUNCATED_KEY_TARGET_SIZE - (TRUNCATED_KEY_SEPARATOR.length + digest.length) if namespace.nil?
+
+      # For historical reasons, truncated keys with namespaces had a length of 250 rather
+      # than 249
+      TRUNCATED_KEY_TARGET_SIZE + 1 - (TRUNCATED_KEY_SEPARATOR.length + digest.length)
+    end
+  end
+end