dalli 1.1.4 → 2.7.11

data/lib/dalli.rb

@@ -1,15 +1,12 @@
+# frozen_string_literal: true
+require 'dalli/compressor'
 require 'dalli/client'
 require 'dalli/ring'
 require 'dalli/server'
 require 'dalli/socket'
 require 'dalli/version'
 require 'dalli/options'
-
-unless ''.respond_to?(:bytesize)
-  class String
-    alias_method :bytesize, :size
-  end
-end
+require 'dalli/railtie' if defined?(::Rails::Railtie)
 
 module Dalli
   # generic error
@@ -18,8 +15,12 @@ module Dalli
   class NetworkError < DalliError; end
   # no server available/alive error
   class RingError < DalliError; end
-  # application error in marshalling
+  # application error in marshalling serialization
   class MarshalError < DalliError; end
+  # application error in marshalling deserialization or decompression
+  class UnmarshalError < DalliError; end
+  # payload too big for memcached
+  class ValueOverMaxSize < DalliError; end
 
   def self.logger
     @logger ||= (rails_logger || default_logger)
@@ -40,6 +41,7 @@ module Dalli
   def self.logger=(logger)
     @logger = logger
   end
+
 end
 
 if defined?(RAILS_VERSION) && RAILS_VERSION < '3'

data/lib/dalli/cas/client.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+require 'dalli/client'
+
+module Dalli
+  class Client
+    ##
+    # Get the value and CAS ID associated with the key.  If a block is provided,
+    # value and CAS will be passed to the block.
+    def get_cas(key)
+      (value, cas) = perform(:cas, key)
+      value = (!value || value == 'Not found') ? nil : value
+      if block_given?
+        yield value, cas
+      else
+        [value, cas]
+      end
+    end
+
+    ##
+    # Fetch multiple keys efficiently, including available metadata such as CAS.
+    # If a block is given, yields key/data pairs one a time.  Data is an array:
+    # [value, cas_id]
+    # If no block is given, returns a hash of
+    #   { 'key' => [value, cas_id] }
+    def get_multi_cas(*keys)
+      if block_given?
+        get_multi_yielder(keys) {|*args| yield(*args)}
+      else
+        Hash.new.tap do |hash|
+          get_multi_yielder(keys) {|k, data| hash[k] = data}
+        end
+      end
+    end
+
+    ##
+    # Set the key-value pair, verifying existing CAS.
+    # Returns the resulting CAS value if succeeded, and falsy otherwise.
+    def set_cas(key, value, cas, ttl=nil, options=nil)
+      ttl ||= @options[:expires_in].to_i
+      perform(:set, key, value, ttl, cas, options)
+    end
+
+    ##
+    # Conditionally add a key/value pair, verifying existing CAS, only if the
+    # key already exists on the server.  Returns the new CAS value if the
+    # operation succeeded, or falsy otherwise.
+    def replace_cas(key, value, cas, ttl=nil, options=nil)
+      ttl ||= @options[:expires_in].to_i
+      perform(:replace, key, value, ttl, cas, options)
+    end
+
+    # Delete a key/value pair, verifying existing CAS.
+    # Returns true if succeeded, and falsy otherwise.
+    def delete_cas(key, cas=0)
+      perform(:delete, key, cas)
+    end
+
+  end
+end

data/lib/dalli/client.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+require 'digest/md5'
+require 'set'
+
 # encoding: ascii
 module Dalli
   class Client
@@ -6,48 +10,33 @@ module Dalli
     # Dalli::Client is the main class which developers will use to interact with
     # the memcached server.  Usage:
     #
-    #   Dalli::Client.new(['localhost:11211:10', 'cache-2.example.com:11211:5', '192.168.0.1:22122:5'],
+    #   Dalli::Client.new(['localhost:11211:10', 'cache-2.example.com:11211:5', '192.168.0.1:22122:5', '/var/run/memcached/socket'],
     #                   :threadsafe => true, :failover => true, :expires_in => 300)
     #
     # servers is an Array of "host:port:weight" where weight allows you to distribute cache unevenly.
-    # Both weight and port are optional.  If you pass in nil, Dalli will default to 'localhost:11211'.
-    # Note that the <tt>MEMCACHE_SERVERS</tt> environment variable will override the servers parameter for use
-    # in managed environments like Heroku.
-    #
-    # You can also provide a Unix socket as an argument, for example:
-    #
-    #   Dalli::Client.new("/tmp/memcached.sock")
-    #
-    # Initial testing shows that Unix sockets are about twice as fast as TCP sockets
-    # but Unix sockets only work on localhost.
+    # Both weight and port are optional.  If you pass in nil, Dalli will use the <tt>MEMCACHE_SERVERS</tt>
+    # environment variable or default to 'localhost:11211' if it is not present.  Dalli also supports
+    # the ability to connect to Memcached on localhost through a UNIX socket.  To use this functionality,
+    # use a full pathname (beginning with a slash character '/') in place of the "host:port" pair in
+    # the server configuration.
     #
     # Options:
+    # - :namespace - prepend each key with this value to provide simple namespacing.
     # - :failover - if a server is down, look for and store values on another server in the ring.  Default: true.
     # - :threadsafe - ensure that only one thread is actively using a socket at a time. Default: true.
     # - :expires_in - default TTL in seconds if you do not pass TTL as a parameter to an individual operation, defaults to 0 or forever
-    # - :compression - defaults to false, if true Dalli will compress values larger than 100 bytes before
-    #   sending them to memcached.
-    # - :async - assume its running inside the EM reactor. Requires em-synchrony to be installed. Default: false.
+    # - :compress - defaults to false, if true Dalli will compress values larger than 1024 bytes before sending them to memcached.
+    # - :serializer - defaults to Marshal
+    # - :compressor - defaults to zlib
+    # - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for #fetch operations.
+    # - :digest_class - defaults to Digest::MD5, allows you to pass in an object that responds to the hexdigest method, useful for injecting a FIPS compliant hash object.
     #
     def initialize(servers=nil, options={})
-      @servers = env_servers || servers || '127.0.0.1:11211'
-      @options = { :expires_in => 0 }.merge(options)
-      self.extend(Dalli::Client::MemcacheClientCompatibility) if Dalli::Client.compatibility_mode
+      @servers = normalize_servers(servers || ENV["MEMCACHE_SERVERS"] || '127.0.0.1:11211')
+      @options = normalize_options(options)
       @ring = nil
     end
 
-    ##
-    # Turn on compatibility mode, which mixes in methods in memcache_client_compatibility.rb
-    # This value is set to true in memcache-client.rb.
-    def self.compatibility_mode
-      @compatibility_mode ||= false
-    end
-
-    def self.compatibility_mode=(compatibility_mode)
-      require 'dalli/compatibility'
-      @compatibility_mode = compatibility_mode
-    end
-
     #
     # The standard memcached instruction set
     #
@@ -64,50 +53,50 @@ module Dalli
       Thread.current[:dalli_multi] = old
     end
 
+    ##
+    # Get the value associated with the key.
+    # If a value is not found, then +nil+ is returned.
     def get(key, options=nil)
-      resp = perform(:get, key)
-      (!resp || resp == 'Not found') ? nil : resp
+      perform(:get, key, options)
     end
 
     ##
     # Fetch multiple keys efficiently.
-    # Returns a hash of { 'key' => 'value', 'key2' => 'value1' }
+    # If a block is given, yields key/value pairs one at a time.
+    # Otherwise returns a hash of { 'key' => 'value', 'key2' => 'value1' }
     def get_multi(*keys)
-      return {} if keys.empty?
-      options = nil
-      options = keys.pop if keys.last.is_a?(Hash) || keys.last.nil?
-      ring.lock do
-        keys.flatten.each do |key|
-          begin
-            perform(:getkq, key)
-          rescue DalliError, NetworkError => e
-            Dalli.logger.debug { e.message }
-            Dalli.logger.debug { "unable to get key #{key}" }
-          end
-        end
+      check_keys = keys.flatten
+      check_keys.compact!
 
-        values = {}
-        ring.servers.each do |server|
-          next unless server.alive?
-          begin
-            server.request(:noop).each_pair do |key, value|
-              values[key_without_namespace(key)] = value
-            end
-          rescue DalliError, NetworkError => e
-            Dalli.logger.debug { e.message }
-            Dalli.logger.debug { "results from this server will be missing" }
-          end
+      return {} if check_keys.empty?
+      if block_given?
+        get_multi_yielder(keys) {|k, data| yield k, data.first}
+      else
+        Hash.new.tap do |hash|
+          get_multi_yielder(keys) {|k, data| hash[k] = data.first}
         end
-        values
       end
     end
 
+    CACHE_NILS = {cache_nils: true}.freeze
+
+    # Fetch the value associated with the key.
+    # If a value is found, then it is returned.
+    #
+    # If a value is not found and no block is given, then nil is returned.
+    #
+    # If a value is not found (or if the found value is nil and :cache_nils is false)
+    # and a block is given, the block will be invoked and its return value
+    # written to the cache and returned.
     def fetch(key, ttl=nil, options=nil)
-      ttl ||= @options[:expires_in]
+      options = options.nil? ? CACHE_NILS : options.merge(CACHE_NILS) if @options[:cache_nils]
       val = get(key, options)
-      if val.nil? && block_given?
+      not_found = @options[:cache_nils] ?
+        val == Dalli::Server::NOT_FOUND :
+        val.nil?
+      if not_found && block_given?
         val = yield
-        add(key, val, ttl, options)
+        add(key, val, ttl_or_default(ttl), options)
       end
       val
     end
@@ -124,39 +113,40 @@ module Dalli
     # - false if the value was changed by someone else.
     # - true if the value was successfully updated.
     def cas(key, ttl=nil, options=nil, &block)
-      ttl ||= @options[:expires_in]
-      (value, cas) = perform(:cas, key)
-      value = (!value || value == 'Not found') ? nil : value
-      if value
-        newvalue = block.call(value)
-        perform(:set, key, newvalue, ttl, cas, options)
-      end
+      cas_core(key, false, ttl, options, &block)
+    end
+
+    ##
+    # like #cas, but will yield to the block whether or not the value
+    # already exists.
+    #
+    # Returns:
+    # - false if the value was changed by someone else.
+    # - true if the value was successfully updated.
+    def cas!(key, ttl=nil, options=nil, &block)
+      cas_core(key, true, ttl, options, &block)
     end
 
     def set(key, value, ttl=nil, options=nil)
-      raise "Invalid API usage, please require 'dalli/memcache-client' for compatibility, see Upgrade.md" if options == true
-      ttl ||= @options[:expires_in]
-      perform(:set, key, value, ttl, 0, options)
+      perform(:set, key, value, ttl_or_default(ttl), 0, options)
     end
 
     ##
     # Conditionally add a key/value pair, if the key does not already exist
-    # on the server.  Returns true if the operation succeeded.
+    # on the server.  Returns truthy if the operation succeeded.
     def add(key, value, ttl=nil, options=nil)
-      ttl ||= @options[:expires_in]
-      perform(:add, key, value, ttl, options)
+      perform(:add, key, value, ttl_or_default(ttl), options)
     end
 
     ##
     # Conditionally add a key/value pair, only if the key already exists
-    # on the server.  Returns true if the operation succeeded.
+    # on the server.  Returns truthy if the operation succeeded.
     def replace(key, value, ttl=nil, options=nil)
-      ttl ||= @options[:expires_in]
-      perform(:replace, key, value, ttl, options)
+      perform(:replace, key, value, ttl_or_default(ttl), 0, options)
     end
 
     def delete(key)
-      perform(:delete, key)
+      perform(:delete, key, 0)
     end
 
     ##
@@ -178,12 +168,11 @@ module Dalli
       ring.servers.map { |s| s.request(:flush, time += delay) }
     end
 
-    # deprecated, please use #flush.
     alias_method :flush_all, :flush
 
     ##
     # Incr adds the given amount to the counter on the memcached server.
-    # Amt must be a positive value.
+    # Amt must be a positive integer value.
     #
     # If default is nil, the counter must already exist or the operation
     # will fail and will return nil.  Otherwise this method will return
@@ -194,13 +183,12 @@ module Dalli
     # #cas.
     def incr(key, amt=1, ttl=nil, default=nil)
       raise ArgumentError, "Positive values only: #{amt}" if amt < 0
-      ttl ||= @options[:expires_in]
-      perform(:incr, key, amt, ttl, default)
+      perform(:incr, key, amt.to_i, ttl_or_default(ttl), default)
     end
 
     ##
     # Decr subtracts the given amount from the counter on the memcached server.
-    # Amt must be a positive value.
+    # Amt must be a positive integer value.
     #
     # memcached counters are unsigned and cannot hold negative values.  Calling
     # decr on a counter which is 0 will just return 0.
@@ -214,17 +202,51 @@ module Dalli
     # #cas.
     def decr(key, amt=1, ttl=nil, default=nil)
       raise ArgumentError, "Positive values only: #{amt}" if amt < 0
-      ttl ||= @options[:expires_in]
-      perform(:decr, key, amt, ttl, default)
+      perform(:decr, key, amt.to_i, ttl_or_default(ttl), default)
+    end
+
+    ##
+    # Touch updates expiration time for a given key.
+    #
+    # Returns true if key exists, otherwise nil.
+    def touch(key, ttl=nil)
+      resp = perform(:touch, key, ttl_or_default(ttl))
+      resp.nil? ? nil : true
     end
 
     ##
     # Collect the stats for each server.
+    # You can optionally pass a type including :items, :slabs or :settings to get specific stats
     # Returns a hash like { 'hostname:port' => { 'stat1' => 'value1', ... }, 'hostname2:port' => { ... } }
-    def stats
+    def stats(type=nil)
+      type = nil if ![nil, :items,:slabs,:settings].include? type
       values = {}
       ring.servers.each do |server|
-        values["#{server.hostname}:#{server.port}"] = server.alive? ? server.request(:stats) : nil
+        values["#{server.name}"] = server.alive? ? server.request(:stats,type.to_s) : nil
+      end
+      values
+    end
+
+    ##
+    # Reset stats for each server.
+    def reset_stats
+      ring.servers.map do |server|
+        server.alive? ? server.request(:reset_stats) : nil
+      end
+    end
+
+    ##
+    ## Make sure memcache servers are alive, or raise an Dalli::RingError
+    def alive!
+      ring.server_for_key("")
+    end
+
+    ##
+    ## Version of the memcache servers.
+    def version
+      values = {}
+      ring.servers.each do |server|
+        values["#{server.name}"] = server.alive? ? server.request(:version) : nil
       end
       values
     end
@@ -240,49 +262,222 @@ module Dalli
     end
     alias_method :reset, :close
 
+    # Stub method so a bare Dalli client can pretend to be a connection pool.
+    def with
+      yield self
+    end
+
     private
 
+    def cas_core(key, always_set, ttl=nil, options=nil)
+      (value, cas) = perform(:cas, key)
+      value = (!value || value == 'Not found') ? nil : value
+      return if value.nil? && !always_set
+      newvalue = yield(value)
+      perform(:set, key, newvalue, ttl_or_default(ttl), cas, options)
+    end
+
+    def ttl_or_default(ttl)
+      (ttl || @options[:expires_in]).to_i
+    rescue NoMethodError
+      raise ArgumentError, "Cannot convert ttl (#{ttl}) to an integer"
+    end
+
+    def groups_for_keys(*keys)
+      groups = mapped_keys(keys).flatten.group_by do |key|
+        begin
+          ring.server_for_key(key)
+        rescue Dalli::RingError
+          Dalli.logger.debug { "unable to get key #{key}" }
+          nil
+        end
+      end
+      return groups
+    end
+
+    def mapped_keys(keys)
+      keys_array = keys.flatten
+      keys_array.map! { |a| validate_key(a.to_s) }
+      keys_array
+    end
+
+    def make_multi_get_requests(groups)
+      groups.each do |server, keys_for_server|
+        begin
+          # TODO: do this with the perform chokepoint?
+          # But given the fact that fetching the response doesn't take place
+          # in that slot it's misleading anyway. Need to move all of this method
+          # into perform to be meaningful
+          server.request(:send_multiget, keys_for_server)
+        rescue DalliError, NetworkError => e
+          Dalli.logger.debug { e.inspect }
+          Dalli.logger.debug { "unable to get keys for server #{server.name}" }
+        end
+      end
+    end
+
+    def perform_multi_response_start(servers)
+      servers.each do |server|
+        next unless server.alive?
+        begin
+          server.multi_response_start
+        rescue DalliError, NetworkError => e
+          Dalli.logger.debug { e.inspect }
+          Dalli.logger.debug { "results from this server will be missing" }
+          servers.delete(server)
+        end
+      end
+      servers
+    end
+
+    ##
+    # Normalizes the argument into an array of servers.
+    # If the argument is a string, or an array containing strings, it's expected that the URIs are comma separated e.g.
+    # "memcache1.example.com:11211,memcache2.example.com:11211,memcache3.example.com:11211"
+    def normalize_servers(servers)
+      Array(servers).flat_map do |server|
+        if server.is_a? String
+          server.split(",")
+        else
+          server
+        end
+      end
+    end
+
     def ring
       @ring ||= Dalli::Ring.new(
-        Array(@servers).map do |s|
-          Dalli::Server.new(s, @options)
+        @servers.map do |s|
+         server_options = {}
+          if s =~ %r{\Amemcached://}
+            uri = URI.parse(s)
+            server_options[:username] = uri.user
+            server_options[:password] = uri.password
+            s = "#{uri.host}:#{uri.port}"
+          end
+          Dalli::Server.new(s, @options.merge(server_options))
         end, @options
       )
     end
 
-    def env_servers
-      ENV['MEMCACHE_SERVERS'] ? ENV['MEMCACHE_SERVERS'].split(',') : nil
-    end
-
     # Chokepoint method for instrumentation
-    def perform(op, key, *args)
+    def perform(*all_args)
+      return yield if block_given?
+      op, key, *args = *all_args
+
       key = key.to_s
-      validate_key(key)
-      key = key_with_namespace(key)
+      key = validate_key(key)
       begin
         server = ring.server_for_key(key)
-        server.request(op, key, *args)
+        ret = server.request(op, key, *args)
+        ret
       rescue NetworkError => e
-        Dalli.logger.debug { e.message }
+        Dalli.logger.debug { e.inspect }
         Dalli.logger.debug { "retrying request with new server" }
         retry
       end
     end
 
     def validate_key(key)
-      raise ArgumentError, "illegal character in key #{key}" if key.respond_to?(:ascii_only?) && !key.ascii_only?
-      raise ArgumentError, "illegal character in key #{key}" if key =~ /\s/
-      raise ArgumentError, "illegal character in key #{key}" if key =~ /[\x00-\x20\x80-\xFF]/
-      raise ArgumentError, "key cannot be blank" if key.nil? || key.strip.size == 0
-      raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
+      raise ArgumentError, "key cannot be blank" if !key || key.length == 0
+      key = key_with_namespace(key)
+      if key.length > 250
+        digest_class = @options[:digest_class] || ::Digest::MD5
+        max_length_before_namespace = 212 - (namespace || '').size
+        key = "#{key[0, max_length_before_namespace]}:md5:#{digest_class.hexdigest(key)}"
+      end
+      return key
     end
 
     def key_with_namespace(key)
-      @options[:namespace] ? "#{@options[:namespace]}:#{key}" : key
+      (ns = namespace) ? "#{ns}:#{key}" : key
     end
 
     def key_without_namespace(key)
-      @options[:namespace] ? key.gsub(%r(\A#{@options[:namespace]}:), '') : key
+      (ns = namespace) ? key.sub(%r(\A#{Regexp.escape ns}:), '') : key
     end
+
+    def namespace
+      return nil unless @options[:namespace]
+      @options[:namespace].is_a?(Proc) ? @options[:namespace].call.to_s : @options[:namespace].to_s
+    end
+
+    def normalize_options(opts)
+      if opts[:compression]
+        Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just :compress => true.  Please update your configuration."
+        opts[:compress] = opts.delete(:compression)
+      end
+      begin
+        opts[:expires_in] = opts[:expires_in].to_i if opts[:expires_in]
+      rescue NoMethodError
+        raise ArgumentError, "cannot convert :expires_in => #{opts[:expires_in].inspect} to an integer"
+      end
+      if opts[:digest_class] && !opts[:digest_class].respond_to?(:hexdigest)
+        raise ArgumentError, "The digest_class object must respond to the hexdigest method"
+      end
+      opts
+    end
+
+    ##
+    # Yields, one at a time, keys and their values+attributes.
+    def get_multi_yielder(keys)
+      perform do
+        return {} if keys.empty?
+        ring.lock do
+          begin
+            groups = groups_for_keys(keys)
+            if unfound_keys = groups.delete(nil)
+              Dalli.logger.debug { "unable to get keys for #{unfound_keys.length} keys because no matching server was found" }
+            end
+            make_multi_get_requests(groups)
+
+            servers = groups.keys
+            return if servers.empty?
+            servers = perform_multi_response_start(servers)
+
+            start = Time.now
+            while true
+              # remove any dead servers
+              servers.delete_if { |s| s.sock.nil? }
+              break if servers.empty?
+
+              # calculate remaining timeout
+              elapsed = Time.now - start
+              timeout = servers.first.options[:socket_timeout]
+              time_left = (elapsed > timeout) ? 0 : timeout - elapsed
+
+              sockets = servers.map(&:sock)
+              readable, _ = IO.select(sockets, nil, nil, time_left)
+
+              if readable.nil?
+                # no response within timeout; abort pending connections
+                servers.each do |server|
+                  Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
+                  server.multi_response_abort
+                end
+                break
+
+              else
+                readable.each do |sock|
+                  server = sock.server
+
+                  begin
+                    server.multi_response_nonblock.each_pair do |key, value_list|
+                      yield key_without_namespace(key), value_list
+                    end
+
+                    if server.multi_response_completed?
+                      servers.delete(server)
+                    end
+                  rescue NetworkError
+                    servers.delete(server)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
   end
 end