dalli 5.0.10 → 5.1.0

data/lib/dalli/compressor.rb

@@ -13,54 +13,8 @@ module Dalli
       Zlib::Deflate.deflate(data)
     end
 
-    # DEFLATE can't expand data by more than about 1032:1
-    MAX_EXPANSION = 1032
-    # Compressed input is fed to zlib this many bytes at a time. MRI's zlib
-    # yields the output in small chunks; one that returns it instead still
-    # produces at most MAX_EXPANSION times this per step.
-    INFLATE_SLICE_BYTES = 64 * 1024
-
-    # max_bytes caps the decompressed size; inflating stops and raises as soon
-    # as the output passes it, so a small compressed value can't expand into
-    # gigabytes of memory.
-    def self.decompress(data, max_bytes: nil)
-      return Zlib::Inflate.inflate(data) unless may_exceed?(data, max_bytes)
-
-      inflate_within_limit(Zlib::Inflate.new, data, max_bytes)
-    end
-
-    # False when even the largest possible expansion stays within max_bytes,
-    # so the value can be inflated in one step
-    def self.may_exceed?(data, max_bytes)
-      max_bytes ? data.bytesize * MAX_EXPANSION > max_bytes : false
-    end
-
-    def self.inflate_within_limit(inflater, data, max_bytes)
-      out = String.new(encoding: Encoding::BINARY)
-      0.step(data.bytesize - 1, INFLATE_SLICE_BYTES) do |pos|
-        inflate_slice(inflater, data.byteslice(pos, INFLATE_SLICE_BYTES), out, max_bytes)
-        # Anything after the end of the stream isn't part of the value, as
-        # with Zlib::Inflate.inflate; finish would return it as output
-        break if inflater.finished?
-      end
-      return out if inflater.finished?
-
-      append_within_limit(out, inflater.finish, max_bytes)
-    ensure
-      inflater.close
-    end
-
-    # Both yielded and returned output count toward the limit
-    def self.inflate_slice(inflater, slice, out, max_bytes)
-      rest = inflater.inflate(slice) { |chunk| append_within_limit(out, chunk, max_bytes) }
-      append_within_limit(out, rest, max_bytes)
-    end
-
-    def self.append_within_limit(out, chunk, max_bytes)
-      out << chunk if chunk
-      return out if out.bytesize <= max_bytes
-
-      raise Dalli::UnmarshalError, "Decompressed value exceeds #{max_bytes} bytes (decompressed_max_bytes)"
+    def self.decompress(data)
+      Zlib::Inflate.inflate(data)
     end
   end
 
@@ -78,11 +32,9 @@ module Dalli
       io.string
     end
 
-    def self.decompress(data, max_bytes: nil)
-      return Zlib::GzipReader.new(StringIO.new(data, 'rb')).read unless Compressor.may_exceed?(data, max_bytes)
-
-      # MAX_WBITS + 16 makes zlib read the gzip format (and check its CRC)
-      Compressor.inflate_within_limit(Zlib::Inflate.new(Zlib::MAX_WBITS + 16), data, max_bytes)
+    def self.decompress(data)
+      io = StringIO.new(data, 'rb')
+      Zlib::GzipReader.new(io).read
     end
   end
 end

data/lib/dalli/key_manager.rb

@@ -58,7 +58,7 @@ module Dalli
       raise ArgumentError, 'key cannot be blank' unless key&.length&.positive?
 
       key = key_with_namespace(key)
-      wire_length(key) > MAX_KEY_LENGTH ? truncated_key(key) : key
+      key.length > MAX_KEY_LENGTH ? truncated_key(key) : key
     end
 
     ##
@@ -122,28 +122,7 @@ module Dalli
     ##
     def truncated_key(key)
       digest = digest_class.hexdigest(key)
-      prefix = key[0, [prefix_length(digest), 0].max]
-      truncated = "#{prefix}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
-      # A prefix with whitespace or non-ASCII characters sends the whole key
-      # base64-encoded, which can still be too long on the wire; shorten it.
-      while wire_length(truncated) > MAX_KEY_LENGTH && !prefix.empty?
-        prefix = prefix[0...-1]
-        truncated = "#{prefix}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
-      end
-      return truncated if wire_length(truncated) <= MAX_KEY_LENGTH
-
-      # Only possible with a digest_class whose digests aren't short hex strings
-      raise ArgumentError, "#{digest_class} digests are too long to fit in a truncated key"
-    end
-
-    # memcached's key limit applies to the key as sent: in bytes, and after
-    # base64 encoding when the key needs it. A key can be within the limit in
-    # characters and still be rejected, and memcached's CLIENT_ERROR for it
-    # must never reach a pipelined get.
-    def wire_length(key)
-      return key.bytesize unless Dalli::Protocol::Meta::KeyRegularizer.required?(key)
-
-      ((key.bytesize + 2) / 3) * 4
+      "#{key[0, prefix_length(digest)]}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
     end
 
     def prefix_length(digest)

data/lib/dalli/pipelined_deleter.rb

@@ -15,17 +15,24 @@ module Dalli
     ##
     # Deletes multiple keys from memcached.
     #
+    # `req_options` is applied to every delete in the batch (see
+    # Dalli::Client#delete for the supported meta-delete keys).
+    #
     # @param keys [Array<String>] keys to delete
-    # @return [Integer] the number of keys that were deleted. This is
-    #   best-effort: on a network error the operation is retried, and keys
-    #   deleted before the error are not recounted, so the result may
-    #   under-report the number actually removed when a failure occurs.
+    # @return [Integer] the number of keys the server found and acted on. Only
+    #   a key that did not exist decrements this count, so with
+    #   `req_options: {invalidate: true}` it reports how many keys were
+    #   tombstoned rather than removed. This is best-effort: a transient
+    #   network error is retried automatically, and keys handled before the
+    #   error are not recounted, so the result may under-report when a retry
+    #   occurs. If a server remains unreachable after retrying, raises
+    #   Dalli::NetworkError.
     ##
-    def process(keys)
+    def process(keys, req_options = nil)
       return 0 if keys.empty?
 
       @ring.lock do
-        groups = setup_requests(keys)
+        groups = setup_requests(keys, req_options)
         finish_requests(groups)
       end
     rescue Dalli::RetryableNetworkError => e
@@ -36,9 +43,9 @@ module Dalli
 
     private
 
-    def setup_requests(keys)
+    def setup_requests(keys, req_options = nil)
       groups = groups_for_keys(keys)
-      make_delete_requests(groups)
+      make_delete_requests(groups, req_options)
       groups
     end
 
@@ -46,12 +53,20 @@ module Dalli
     # Loop through the server-grouped sets of keys, writing
     # the corresponding quiet delete requests to the appropriate servers
     ##
-    def make_delete_requests(groups)
+    # NetworkError (which RetryableNetworkError subclasses) must propagate: the
+    # top-level rescue in #process retries the whole pipelined delete on it. A
+    # combined `rescue DalliError, NetworkError` would silently swallow
+    # RetryableNetworkError too -- since NetworkError < DalliError -- dropping
+    # this server's keys on a transient hiccup instead of retrying. Only a
+    # non-network DalliError should be swallowed here.
+    def make_delete_requests(groups, req_options = nil)
       groups.each do |server, keys_for_server|
         keys_for_server.select! do |key|
-          server.request(:pipelined_delete, key)
+          server.request(:pipelined_delete, key, req_options)
           true
-        rescue DalliError, NetworkError => e
+        rescue Dalli::NetworkError
+          raise
+        rescue DalliError => e
           Dalli.logger.debug { e.inspect }
           Dalli.logger.debug { "unable to delete key #{key} for server #{server.name}" }
           false
@@ -66,7 +81,9 @@ module Dalli
     def finish_requests(groups)
       groups.sum do |server, keys_for_server|
         server.request(:finish_pipelined_delete, keys_for_server.size)
-      rescue DalliError, NetworkError => e
+      rescue Dalli::NetworkError
+        raise
+      rescue DalliError => e
         Dalli.logger.debug { e.inspect }
         Dalli.logger.debug { "unable to complete pipelined delete on server #{server.name}" }
         0

data/lib/dalli/pipelined_getter.rb

@@ -20,13 +20,18 @@ module Dalli
     ##
     # Yields, one at a time, keys and their values+attributes.
     #
-    def process(keys, &block)
+    # `req_options` accepts :p_token/:l_token, applied to every key in the batch.
+    #
+    # A transient network error is retried automatically. If a server remains
+    # unreachable after retrying, raises Dalli::NetworkError.
+    #
+    def process(keys, req_options = nil, &block)
       return {} if keys.empty?
 
       @ring.lock do
         # Stores partial results collected during interleaved send phase
         @partial_results = {}
-        servers = setup_requests(keys)
+        servers = setup_requests(keys, req_options)
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
         # First yield any partial results collected during interleaved send
@@ -40,6 +45,40 @@ module Dalli
       retry
     end
 
+    ##
+    # Stale-aware bulk get across servers.  Returns { key => metadata Hash } for
+    # the keys that were found; see Protocol::Meta#read_multi_with_metadata_req
+    # for why misses are absent rather than present with miss: true.
+    #
+    # Unlike #process this issues one request per server and reads its full
+    # response before moving on, rather than pipelining across servers: the
+    # metadata path has no interleaving support, and the stale-aware callers it
+    # serves fetch far smaller batches than get_multi does.
+    #
+    # `req_options` accepts :p_token/:l_token, applied to every key in the batch.
+    #
+    def process_with_metadata(keys, req_options = nil)
+      return {} if keys.empty?
+
+      @ring.lock do
+        results = {}
+        groups_for_keys(keys).each do |server, keys_for_server|
+          results.merge!(server.request(:read_multi_with_metadata_req, keys_for_server, req_options))
+        rescue Dalli::RetryableNetworkError
+          raise
+        rescue DalliError, NetworkError => e
+          Dalli.logger.debug { e.inspect }
+          Dalli.logger.debug { "unable to get keys for server #{server.name}" }
+        end
+        results.transform_keys! { |key| @key_manager.key_without_namespace(key) }
+        results
+      end
+    rescue Dalli::RetryableNetworkError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { 'retrying pipelined get with metadata because of network error' }
+      retry
+    end
+
     private
 
     def yield_partial_results
@@ -49,9 +88,9 @@ module Dalli
       @partial_results.clear
     end
 
-    def setup_requests(keys)
+    def setup_requests(keys, req_options = nil)
       groups = groups_for_keys(keys)
-      make_getkq_requests(groups)
+      make_getkq_requests(groups, req_options)
 
       # TODO: How does this exit on a NetworkError
       finish_queries(groups.keys)
@@ -65,17 +104,27 @@ module Dalli
     # on the wire by switching from getkq to getq, and using
     # the opaque value to match requests to responses.
     ##
-    def make_getkq_requests(groups)
+    def make_getkq_requests(groups, req_options = nil)
       groups.each do |server, keys_for_server|
         if keys_for_server.size <= INTERLEAVE_THRESHOLD
           # Small batch - send all at once (existing behavior)
-          server.request(:pipelined_get, keys_for_server)
+          server.request(:pipelined_get, keys_for_server, req_options)
         else
           # Large batch - interleave sends with response draining
           # Pass @partial_results directly to avoid hash allocation/merge overhead
-          server.request(:pipelined_get_interleaved, keys_for_server, CHUNK_SIZE, @partial_results)
+          server.request(:pipelined_get_interleaved, keys_for_server, CHUNK_SIZE, @partial_results, req_options)
         end
-      rescue DalliError, NetworkError => e
+      # NetworkError (which RetryableNetworkError subclasses) must propagate:
+      # #process's top-level rescue retries the whole pipelined get on it. This
+      # rescue used to catch DalliError and NetworkError together -- since
+      # NetworkError < DalliError, that silently swallowed RetryableNetworkError
+      # too, dropping this server's keys from the result on a transient hiccup
+      # instead of retrying, with nothing surfaced above debug-level logging.
+      # Only a non-network DalliError (this server genuinely can't serve these
+      # keys) should be swallowed here.
+      rescue Dalli::NetworkError
+        raise
+      rescue DalliError => e
         Dalli.logger.debug { e.inspect }
         Dalli.logger.debug { "unable to get keys for server #{server.name}" }
       end

data/lib/dalli/pipelined_setter.rb

@@ -14,7 +14,9 @@ module Dalli
 
     ##
     # Writes multiple key-value pairs to memcached.
-    # Raises an error if any server is unavailable.
+    #
+    # A transient network error is retried automatically. If a server remains
+    # unreachable after retrying, raises Dalli::NetworkError.
     #
     # @param hash [Hash] key-value pairs to set
     # @param ttl [Integer] time-to-live in seconds
@@ -46,13 +48,21 @@ module Dalli
     # Loop through the server-grouped sets of keys, writing
     # the corresponding quiet set requests to the appropriate servers
     ##
+    # NetworkError (which RetryableNetworkError subclasses) must propagate: the
+    # top-level rescue in #process retries the whole pipelined set on it. A
+    # combined `rescue DalliError, NetworkError` would silently swallow
+    # RetryableNetworkError too -- since NetworkError < DalliError -- dropping
+    # this server's keys on a transient hiccup instead of retrying. Only a
+    # non-network DalliError should be swallowed here.
     def make_set_requests(groups, hash, ttl, req_options)
       groups.each do |server, keys_for_server|
         keys_for_server.each do |key|
           original_key = @key_manager.key_without_namespace(key)
           value = hash[original_key]
           server.request(:pipelined_set, key, value, ttl, req_options)
-        rescue DalliError, NetworkError => e
+        rescue Dalli::NetworkError
+          raise
+        rescue DalliError => e
           Dalli.logger.debug { e.inspect }
           Dalli.logger.debug { "unable to set key #{key} for server #{server.name}" }
         end
@@ -65,7 +75,9 @@ module Dalli
     def finish_requests(servers)
       servers.each do |server|
         server.request(:noop)
-      rescue DalliError, NetworkError => e
+      rescue Dalli::NetworkError
+        raise
+      rescue DalliError => e
         Dalli.logger.debug { e.inspect }
         Dalli.logger.debug { "unable to complete pipelined set on server #{server.name}" }
       end

data/lib/dalli/protocol/base.rb

@@ -35,14 +35,6 @@ module Dalli
         @raw_mode
       end
 
-      # True when values should come back as stored: the client is in raw mode,
-      # or this request passed raw: true. Retrieval paths use it to skip
-      # requesting flags, so a stored value is never deserialized or
-      # decompressed for a caller who asked for raw bytes.
-      def raw_request?(options)
-        raw_mode? || (options.is_a?(Hash) && options[:raw]) ? true : false
-      end
-
       # Chokepoint method for error handling and ensuring liveness
       def request(opkey, *args)
         verify_state(opkey)
@@ -86,6 +78,19 @@ module Dalli
       # particular memcached instance is available for use.
       def alive?
         ensure_connected!
+      rescue Dalli::RetryableNetworkError => e
+        # A single connection attempt failure is retryable -- the same
+        # contract #request enforces on the send/receive path. Retrying here
+        # lets error_on_request!'s own fail-count threshold decide when to
+        # give up: it keeps raising RetryableNetworkError until
+        # socket_max_failures is reached, then raises a terminal
+        # NetworkError via down!, which the next rescue converts to false.
+        # Without this, a single transient hiccup during the liveness check
+        # itself -- as opposed to an actual request -- would report this
+        # server as not alive even though it would have reconnected fine.
+        Dalli.logger.debug { e.inspect }
+        Dalli.logger.debug { "retrying connection attempt to #{name} because of network error" }
+        retry
       rescue Dalli::NetworkError
         # ensure_connected! raises a NetworkError if connection fails.  We
         # want to capture that error and convert it to a boolean value here.
@@ -121,15 +126,7 @@ module Dalli
         reconnect_on_pipeline_complete!
         values = nil
 
-        # Only the read is socket I/O. The caller's block runs below, outside
-        # this rescue, so an Errno or Timeout::Error it raises reaches the
-        # caller instead of being mistaken for a network failure (which closed
-        # the connection and retried the whole get_multi, yielding keys twice).
-        begin
-          response_buffer.read
-        rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
-          @connection_manager.error_on_request!(e)
-        end
+        response_buffer.read
 
         status, cas, key, value = response_buffer.process_single_getk_response
         # status is not nil only if we have a full response to parse
@@ -155,6 +152,8 @@ module Dalli
         end
 
         values || {}
+      rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
+        @connection_manager.error_on_request!(e)
       end
       # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
@@ -168,13 +167,10 @@ module Dalli
         @connection_manager.abort_request!
         return true unless connected?
 
-        # Closes the connection, which ensures that our connection is in a
-        # clean state for future requests. The get_multi ran out of time
-        # overall rather than hitting a socket error, so it doesn't count
-        # toward socket_max_failures: two slow get_multis in a row would
-        # otherwise mark a healthy server down.
-        Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
-        @connection_manager.close
+        # Closes the connection, which ensures that our connection
+        # is in a clean state for future requests
+        @connection_manager.error_on_request!('External timeout')
+      rescue NetworkError
         true
       end
 
@@ -254,13 +250,44 @@ module Dalli
         opts[:cache_nils] ? true : false
       end
 
+      # Extracts opaque routing-token kwargs (:p_token, :l_token) from a
+      # request-options Hash so they can be splatted into a RequestFormatter
+      # call. Returns {} when neither is set, so the splat is a no-op on the
+      # common path. Validation (type, forbidden bytes) happens at the
+      # wire-formatter level, where it can raise uniformly regardless of how
+      # the token reached the formatter.
+      def routing_token_kwargs(opts)
+        return {} unless opts.is_a?(Hash)
+        return {} unless opts[:p_token] || opts[:l_token]
+
+        { p_token: opts[:p_token], l_token: opts[:l_token] }
+      end
+
+      # Maps the client-facing meta-delete options onto RequestFormatter's flag
+      # names, so they can be splatted into a meta_delete call.  Returns {} when
+      # none are set, keeping the splat a no-op on the common path.
+      #
+      # :tombstone_ttl becomes the T flag, which is the same TTL the formatter
+      # already accepted -- deliberately not a second TTL parameter, since two
+      # would allow emitting two T tokens in one request.  It is sanitized like
+      # every other TTL Dalli sends.
+      def tombstone_kwargs(opts)
+        return {} unless opts.is_a?(Hash)
+
+        kwargs = {}
+        kwargs[:stale] = true if opts[:invalidate]
+        kwargs[:ttl] = TtlSanitizer.sanitize(Integer(opts[:tombstone_ttl])) if opts[:tombstone_ttl]
+        kwargs[:drop_value] = true if opts[:drop_value]
+        kwargs
+      end
+
       def connect
         @connection_manager.establish_connection
         @version = version
         up!
       end
 
-      def pipelined_get(keys)
+      def pipelined_get(keys, options = nil)
         # Clear buffer to remove any stale data from interrupted operations.
         # Use clear (not reset) to keep pipeline_complete? = true, which is
         # the expected state before pipeline_response_setup is called.
@@ -268,7 +295,7 @@ module Dalli
 
         req = +''
         keys.each do |key|
-          req << quiet_get_request(key)
+          req << quiet_get_request(key, options)
         end
         # Could send noop here instead of in pipeline_response_setup
         write(req)
@@ -277,7 +304,7 @@ module Dalli
       # For large batches, interleave writing requests with draining responses.
       # This prevents socket buffer deadlock when sending many keys.
       # Populates the provided results hash with any responses drained during send.
-      def pipelined_get_interleaved(keys, chunk_size, results)
+      def pipelined_get_interleaved(keys, chunk_size, results, options = nil)
         # Initialize the response buffer for draining during send phase
         response_buffer.ensure_ready
 
@@ -285,7 +312,7 @@ module Dalli
           # Build and write this chunk of requests
           req = +''
           chunk.each do |key|
-            req << quiet_get_request(key)
+            req << quiet_get_request(key, options)
           end
           write(req)
           @connection_manager.flush