dalli 5.0.7 → 5.1.0

data/lib/dalli/client.rb

@@ -68,6 +68,7 @@ module Dalli
     # Get the value associated with the key.
     # If a value is not found, then +nil+ is returned.
     def get(key, req_options = nil)
+      validate_routing_tokens!(req_options)
       perform(:get, key, req_options)
     end
 
@@ -75,8 +76,9 @@ module Dalli
     # Gat (get and touch) fetch an item and simultaneously update its expiration time.
     #
     # If a value is not found, then +nil+ is returned.
-    def gat(key, ttl = nil)
-      perform(:gat, key, ttl_or_default(ttl))
+    def gat(key, ttl = nil, req_options = nil)
+      validate_routing_tokens!(req_options)
+      perform(:gat, key, ttl_or_default(ttl), req_options)
     end
 
     ##
@@ -91,8 +93,9 @@ module Dalli
     ##
     # Get the value and CAS ID associated with the key.  If a block is provided,
     # value and CAS will be passed to the block.
-    def get_cas(key)
-      (value, cas) = perform(:cas, key)
+    def get_cas(key, req_options = nil)
+      validate_routing_tokens!(req_options)
+      (value, cas) = perform(:cas, key, req_options)
       return [value, cas] unless block_given?
 
       yield value, cas
@@ -106,17 +109,23 @@ module Dalli
     #   - :return_cas [Boolean] return the CAS value (default: true)
     #   - :return_hit_status [Boolean] return whether item was previously accessed
     #   - :return_last_access [Boolean] return seconds since last access
+    #   - :return_ttl_remaining [Boolean] return seconds of TTL remaining (-1 if no TTL)
     #   - :skip_lru_bump [Boolean] don't bump LRU or update access stats
     #
     # @return [Hash] containing:
     #   - :value - the cached value (or nil on miss)
     #   - :cas - the CAS value
+    #   - :miss - true when the key does not exist.  Always present.  Prefer it
+    #     over a nil :value, which cannot distinguish a miss from a stored nil
+    #     under cache_nils
     #   - :hit_before - true/false if previously accessed (only if return_hit_status: true)
     #   - :last_access - seconds since last access (only if return_last_access: true)
+    #   - :ttl_remaining - seconds of TTL remaining, -1 when the item has no
+    #     expiry (only if return_ttl_remaining: true)
     #
     # @example Get with hit status
     #   result = client.get_with_metadata('key', return_hit_status: true)
-    #   # => { value: "data", cas: 123, hit_before: true }
+    #   # => { value: "data", cas: 123, miss: false, hit_before: true }
     #
     # @example Get with all metadata without affecting LRU
     #   result = client.get_with_metadata('key',
@@ -127,6 +136,7 @@ module Dalli
     #   # => { value: "data", cas: 123, hit_before: true, last_access: 42 }
     #
     def get_with_metadata(key, options = {})
+      validate_routing_tokens!(options)
       key = key.to_s
       key = @key_manager.validate_key(key)
 
@@ -144,32 +154,98 @@ module Dalli
     # Fetch multiple keys efficiently.
     # If a block is given, yields key/value pairs one at a time.
     # Otherwise returns a hash of { 'key' => 'value', 'key2' => 'value1' }
+    #
+    # `req_options` accepts :p_token/:l_token, applied to every key in the batch.
+    #
+    # A transient network error is retried automatically. If a server remains
+    # unreachable after retrying, raises Dalli::NetworkError rather than
+    # silently omitting that server's keys from the result.
+    #
+    # @raise [Dalli::NetworkError] if a server is unreachable after retrying
     # rubocop:disable Style/ExplicitBlockArgument
-    def get_multi(*keys)
+    def get_multi(*keys, req_options: nil)
       keys.flatten!
       keys.compact!
       return {} if keys.empty?
 
+      validate_routing_tokens!(req_options)
+
       if block_given?
-        get_multi_yielding(keys) { |k, v| yield k, v }
+        get_multi_yielding(keys, req_options) { |k, v| yield k, v }
       else
-        get_multi_hash(keys)
+        get_multi_hash(keys, req_options)
       end
     end
     # rubocop:enable Style/ExplicitBlockArgument
 
+    ##
+    # Fetch multiple keys efficiently, returning a stale-aware metadata Hash per
+    # key.  If a block is given, yields key/metadata pairs one at a time.
+    #
+    # Like #get_multi and #get_multi_cas, keys that were not found are omitted
+    # from the result -- absence is the miss.  A tombstoned item (see #delete
+    # with the meta protocol's invalidate flag) is *not* a miss: it is returned
+    # with stale: true, possibly with an empty value, which is the distinction
+    # stale-aware callers need.
+    #
+    #   client.get_multi_with_metadata('a', 'b', 'absent')
+    #   # => { 'a' => { value: 'v', cas: 12, stale: false, miss: false },
+    #   #      'b' => { value: '',  cas: 13, stale: true,  miss: false } }
+    #
+    # Missing keys are `requested - result.keys`.
+    #
+    # Result key order matches request order only when every key lands on the
+    # same server; across multiple servers it follows per-server response
+    # order instead, the same as #get_multi.
+    #
+    # `req_options` accepts :p_token/:l_token, applied to every key in the batch.
+    #
+    # @param keys [Array<String>] the keys to fetch
+    # @param req_options [Hash, nil] routing-token options
+    # @return [Hash] key => { value:, cas:, stale:, miss: }
+    def get_multi_with_metadata(*keys, req_options: nil, &block)
+      keys.flatten!
+      keys.compact!
+      return {} if keys.empty?
+
+      validate_routing_tokens!(req_options)
+
+      results = Instrumentation.trace('get_multi_with_metadata',
+                                      multi_trace_attrs('get_multi_with_metadata', keys.size, keys)) do
+        if ring.servers.size == 1
+          single_server_get_multi_with_metadata(keys, req_options)
+        else
+          pipelined_getter.process_with_metadata(keys, req_options)
+        end
+      end
+
+      if block
+        results.each(&block)
+        # Matches get_multi/get_multi_cas: nil when a block is given, so
+        # callers can't come to depend on a return value that block-form
+        # get_multi never provided.
+        return nil
+      end
+
+      results
+    end
+
     ##
     # Fetch multiple keys efficiently, including available metadata such as CAS.
     # If a block is given, yields key/data pairs one a time.  Data is an array:
     # [value, cas_id]
     # If no block is given, returns a hash of
     #   { 'key' => [value, cas_id] }
-    def get_multi_cas(*keys)
+    #
+    # `req_options` accepts :p_token/:l_token, applied to every key in the batch.
+    def get_multi_cas(*keys, req_options: nil)
+      validate_routing_tokens!(req_options)
+
       if block_given?
-        pipelined_getter.process(keys) { |*args| yield(*args) }
+        pipelined_getter.process(keys, req_options) { |*args| yield(*args) }
       else
         {}.tap do |hash|
-          pipelined_getter.process(keys) { |k, data| hash[k] = data }
+          pipelined_getter.process(keys, req_options) { |k, data| hash[k] = data }
         end
       end
     end
@@ -225,8 +301,7 @@ module Dalli
     def fetch_with_lock(key, ttl: nil, lock_ttl: 30, recache_threshold: nil, req_options: nil, &block)
       raise ArgumentError, 'Block is required for fetch_with_lock' unless block_given?
 
-      validate_integer!(:lock_ttl, lock_ttl)
-      validate_integer!(:recache_threshold, recache_threshold)
+      validate_routing_tokens!(req_options)
       key = key.to_s
       key = @key_manager.validate_key(key)
 
@@ -302,16 +377,24 @@ module Dalli
     # This method is more efficient than calling set() in a loop because
     # it batches requests by server and uses quiet mode.
     #
+    # A transient network error is retried automatically. If a server remains
+    # unreachable after retrying, raises Dalli::NetworkError; keys already
+    # sent to other servers before the error are not rolled back.
+    #
     # @param hash [Hash] key-value pairs to set
     # @param ttl [Integer] time-to-live in seconds (optional, uses default if not provided)
-    # @param req_options [Hash] options passed to each set operation
+    # @param req_options [Hash] options passed to each set operation; accepts
+    #   :p_token/:l_token, applied to every key in the batch
     # @return [void]
+    # @raise [Dalli::NetworkError] if a server is unreachable after retrying
     #
     # Example:
     #   client.set_multi({ 'key1' => 'value1', 'key2' => 'value2' }, 300)
     def set_multi(hash, ttl = nil, req_options = nil)
       return if hash.empty?
 
+      validate_routing_tokens!(req_options)
+
       Instrumentation.trace('set_multi', multi_trace_attrs('set_multi', hash.size, hash.keys)) do
         if ring.servers.size == 1
           single_server_set_multi(hash, ttl_or_default(ttl), req_options)
@@ -325,6 +408,7 @@ module Dalli
     # Set the key-value pair, verifying existing CAS.
     # Returns the resulting CAS value if succeeded, and falsy otherwise.
     def set_cas(key, value, cas, ttl = nil, req_options = nil)
+      validate_routing_tokens!(req_options)
       perform(:set, key, value, ttl_or_default(ttl), cas, req_options)
     end
 
@@ -332,6 +416,7 @@ module Dalli
     # Conditionally add a key/value pair, if the key does not already exist
     # on the server.  Returns truthy if the operation succeeded.
     def add(key, value, ttl = nil, req_options = nil)
+      validate_routing_tokens!(req_options)
       perform(:add, key, value, ttl_or_default(ttl), req_options)
     end
 
@@ -347,17 +432,49 @@ module Dalli
     # key already exists on the server.  Returns the new CAS value if the
     # operation succeeded, or falsy otherwise.
     def replace_cas(key, value, cas, ttl = nil, req_options = nil)
+      validate_routing_tokens!(req_options)
       perform(:replace, key, value, ttl_or_default(ttl), cas, req_options)
     end
 
     # Delete a key/value pair, verifying existing CAS.
     # Returns true if succeeded, and falsy otherwise.
-    def delete_cas(key, cas = 0)
-      perform(:delete, key, cas)
+    # Delete a key, optionally with a CAS check.
+    #
+    # `req_options` accepts the same meta-delete options as #delete.
+    def delete_cas(key, cas = 0, req_options = nil)
+      validate_delete_options!(req_options)
+      validate_routing_tokens!(req_options)
+      perform(:delete, key, cas, req_options)
     end
 
-    def delete(key)
-      delete_cas(key, 0)
+    ##
+    # Delete a key.
+    #
+    # `req_options` may include the memcached meta-delete options:
+    #
+    # - `:invalidate` (Boolean) — mark the item stale instead of removing it.
+    #   This is the tombstone: readers see `stale: true` from
+    #   #get_with_metadata and #get_multi_with_metadata, and the existing value
+    #   is still readable unless `:drop_value` is also set. A tombstoned key is
+    #   *not* a miss, which lets a reader tell "another process is repopulating
+    #   this" apart from "this was never here".
+    # - `:tombstone_ttl` (Integer seconds) — how long the stale marker lives.
+    #   Requires `:invalidate`; memcached only honors the TTL on a delete when
+    #   it accompanies the invalidate flag, so passing it alone raises
+    #   ArgumentError rather than sending a request the server would treat
+    #   differently than intended. Once it elapses, reads see a miss.
+    # - `:drop_value` (Boolean) — remove the item's value but leave the item, so
+    #   a tombstone need not retain the old payload. On its own it is not a
+    #   tombstone: reads are an ordinary hit with an empty value.
+    # - `:p_token`/`:l_token` (String) — opaque routing tokens for an
+    #   intermediate proxy or router; see #get.
+    #
+    #   dc.delete('key', invalidate: true, tombstone_ttl: 30, drop_value: true)
+    #
+    # @param key [String] the key to delete
+    # @param req_options [Hash, nil] meta-delete options
+    def delete(key, req_options = nil)
+      delete_cas(key, 0, req_options)
     end
 
     ##
@@ -365,22 +482,35 @@ module Dalli
     # This method is more efficient than calling delete() in a loop because
     # it batches requests by server and uses quiet mode.
     #
+    # `req_options` accepts the same meta-delete options as #delete and applies
+    # them to every key in the batch.
+    #
     # @param keys [Array<String>] keys to delete
-    # @return [Integer] the number of keys that were found and deleted. This is
-    #   best-effort: on a network error the operation is retried, and keys
-    #   deleted before the error are not recounted, so the result may
-    #   under-report the number actually removed when a failure occurs.
+    # @param req_options [Hash, nil] meta-delete options
+    # @return [Integer] the number of keys the server found and acted on. Only a
+    #   key that did not exist decrements this count, so with `:invalidate` it
+    #   reports how many keys were tombstoned rather than removed -- the action
+    #   is whichever one the caller asked for. This is best-effort: a transient
+    #   network error is retried automatically, and keys handled before the
+    #   error are not recounted, so the result may under-report when a retry
+    #   occurs. If a server remains unreachable after retrying, raises
+    #   Dalli::NetworkError.
+    # @raise [Dalli::NetworkError] if a server is unreachable after retrying
     #
     # Example:
     #   client.delete_multi(['key1', 'key2', 'key3'])
-    def delete_multi(keys)
+    #   client.delete_multi(%w[key1 key2], invalidate: true, tombstone_ttl: 30)
+    def delete_multi(keys, req_options = nil)
       return 0 if keys.empty?
 
+      validate_delete_options!(req_options)
+      validate_routing_tokens!(req_options)
+
       Instrumentation.trace('delete_multi', multi_trace_attrs('delete_multi', keys.size, keys)) do
         if ring.servers.size == 1
-          single_server_delete_multi(keys)
+          single_server_delete_multi(keys, req_options)
         else
-          pipelined_deleter.process(keys)
+          pipelined_deleter.process(keys, req_options)
         end
       end
     end
@@ -388,22 +518,22 @@ module Dalli
     ##
     # Append value to the value already stored on the server for 'key'.
     # Appending only works for values stored with :raw => true.
-    def append(key, value)
-      perform(:append, key, value.to_s)
+    def append(key, value, req_options = nil)
+      validate_routing_tokens!(req_options)
+      perform(:append, key, value.to_s, req_options)
     end
 
     ##
     # Prepend value to the value already stored on the server for 'key'.
     # Prepending only works for values stored with :raw => true.
-    def prepend(key, value)
-      perform(:prepend, key, value.to_s)
+    def prepend(key, value, req_options = nil)
+      validate_routing_tokens!(req_options)
+      perform(:prepend, key, value.to_s, req_options)
     end
 
     ##
     # Incr adds the given amount to the counter on the memcached server.
     # Amt must be a positive integer value.
-    # Default, if given, must be an Integer (or a String of decimal digits);
-    # anything else raises ArgumentError.
     #
     # If default is nil, the counter must already exist or the operation
     # will fail and will return nil.  Otherwise this method will return
@@ -414,18 +544,16 @@ module Dalli
     # #cas.
     #
     # If the value already exists, it must have been set with raw: true
-    def incr(key, amt = 1, ttl = nil, default = nil)
+    def incr(key, amt = 1, ttl = nil, default = nil, req_options = nil)
       check_positive!(amt)
-      validate_integer!(:default, default)
+      validate_routing_tokens!(req_options)
 
-      perform(:incr, key, amt.to_i, ttl_or_default(ttl), default)
+      perform(:incr, key, amt.to_i, ttl_or_default(ttl), default, req_options)
     end
 
     ##
     # Decr subtracts the given amount from the counter on the memcached server.
     # Amt must be a positive integer value.
-    # Default, if given, must be an Integer (or a String of decimal digits);
-    # anything else raises ArgumentError.
     #
     # memcached counters are unsigned and cannot hold negative values.  Calling
     # decr on a counter which is 0 will just return 0.
@@ -439,11 +567,11 @@ module Dalli
     # #cas.
     #
     # If the value already exists, it must have been set with raw: true
-    def decr(key, amt = 1, ttl = nil, default = nil)
+    def decr(key, amt = 1, ttl = nil, default = nil, req_options = nil)
       check_positive!(amt)
-      validate_integer!(:default, default)
+      validate_routing_tokens!(req_options)
 
-      perform(:decr, key, amt.to_i, ttl_or_default(ttl), default)
+      perform(:decr, key, amt.to_i, ttl_or_default(ttl), default, req_options)
     end
 
     ##
@@ -520,6 +648,30 @@ module Dalli
 
     private
 
+    # Raised before the request reaches a server: RequestFormatter enforces the
+    # same rule, but reaching it means unwinding through Protocol::Base#request,
+    # which logs the failure as unexpected and closes the connection.  A caller
+    # passing the wrong options should get a clean ArgumentError and keep its
+    # connection.
+    def validate_delete_options!(req_options)
+      return unless req_options.is_a?(Hash)
+
+      tombstone_ttl = req_options[:tombstone_ttl]
+      return unless tombstone_ttl
+
+      raise ArgumentError, 'tombstone_ttl requires invalidate: true' unless req_options[:invalidate]
+
+      # tombstone_kwargs coerces this with Integer(), deep inside the request
+      # path; validated here first so a bad value raises cleanly instead of
+      # unwinding through Protocol::Base#request, which would close the
+      # connection on the ArgumentError Integer() raises.
+      begin
+        Integer(tombstone_ttl)
+      rescue ArgumentError, TypeError
+        raise ArgumentError, "tombstone_ttl must be an integer, got #{tombstone_ttl.inspect}"
+      end
+    end
+
     def record_hit_miss_metrics(span, key_count, hit_count)
       return unless span
 
@@ -527,10 +679,10 @@ module Dalli
                           'db.memcached.miss_count' => key_count - hit_count)
     end
 
-    def get_multi_yielding(keys)
+    def get_multi_yielding(keys, req_options = nil)
       Instrumentation.trace_with_result('get_multi', get_multi_attributes(keys)) do |span|
         hit_count = 0
-        pipelined_getter.process(keys) do |k, data|
+        pipelined_getter.process(keys, req_options) do |k, data|
           hit_count += 1
           yield k, data.first
         end
@@ -539,13 +691,13 @@ module Dalli
       end
     end
 
-    def get_multi_hash(keys)
+    def get_multi_hash(keys, req_options = nil)
       Instrumentation.trace_with_result('get_multi', get_multi_attributes(keys)) do |span|
         hash = if ring.servers.size == 1
-                 single_server_get_multi(keys)
+                 single_server_get_multi(keys, req_options)
                else
                  {}.tap do |h|
-                   pipelined_getter.process(keys) { |k, data| h[k] = data.first }
+                   pipelined_getter.process(keys, req_options) { |k, data| h[k] = data.first }
                  end
                end
         record_hit_miss_metrics(span, keys.size, hash.size)
@@ -558,11 +710,33 @@ module Dalli
       server if server&.alive?
     end
 
-    def single_server_get_multi(keys)
+    # The three single_server_* fast-path methods below share one contract,
+    # matching the pipelined multi-server path they stand in for: a transient
+    # RetryableNetworkError is retried (bounded implicitly by the server's own
+    # socket_max_failures, same as the pipelined path's retry), and a hard
+    # NetworkError -- the server genuinely unreachable, not just blipping --
+    # propagates to the caller rather than being swallowed into a silently
+    # wrong result. Only server_for_key/single_server finding no live server
+    # to route to at all is still silent, matching Ring#keys_grouped_by_server
+    # dropping a key it can't route on both the single- and multi-server paths.
+    def single_server_get_multi(keys, req_options = nil)
       keys.map! { |k| @key_manager.validate_key(k.to_s) }
       return {} unless (server = single_server)
 
-      result = server.request(:read_multi_req, keys)
+      result = server.request(:read_multi_req, keys, req_options)
+      result.transform_keys! { |k| @key_manager.key_without_namespace(k) }
+      result
+    rescue Dalli::RetryableNetworkError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { 'retrying single-server get_multi because of network error' }
+      retry
+    end
+
+    def single_server_get_multi_with_metadata(keys, req_options = nil)
+      keys.map! { |k| @key_manager.validate_key(k.to_s) }
+      return {} unless (server = single_server)
+
+      result = server.request(:read_multi_with_metadata_req, keys, req_options)
       result.transform_keys! { |k| @key_manager.key_without_namespace(k) }
       result
     rescue Dalli::NetworkError
@@ -574,24 +748,21 @@ module Dalli
       return unless (server = single_server)
 
       server.request(:write_multi_req, pairs, ttl, req_options)
-    rescue Dalli::NetworkError
-      nil
+    rescue Dalli::RetryableNetworkError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { 'retrying single-server set_multi because of network error' }
+      retry
     end
 
-    def single_server_delete_multi(keys)
+    def single_server_delete_multi(keys, req_options = nil)
       validated_keys = keys.map { |k| @key_manager.validate_key(k.to_s) }
       return 0 unless (server = single_server)
 
-      server.request(:delete_multi_req, validated_keys)
+      server.request(:delete_multi_req, validated_keys, req_options)
     rescue Dalli::RetryableNetworkError => e
-      # Mirror the pipelined path: retry transient errors so a momentary blip
-      # still yields a real count. Bounded by the server's socket_max_failures,
-      # after which a hard NetworkError is raised and handled below.
       Dalli.logger.debug { e.inspect }
       Dalli.logger.debug { 'retrying single-server delete_multi because of network error' }
       retry
-    rescue Dalli::NetworkError
-      0
     end
 
     def get_multi_attributes(keys)
@@ -625,21 +796,33 @@ module Dalli
       raise ArgumentError, "Positive values only: #{amt}" if amt.negative?
     end
 
-    # Numeric arguments that become meta protocol flags (GHSA-6wmv-xq9m-fmp7).
-    # RequestFormatter converts them to Integer as the wire-level backstop;
-    # checking here too raises a clean ArgumentError before the request
-    # starts, instead of Protocol::Base#request logging it as unexpected and
-    # closing the connection.
-    def validate_integer!(name, value)
-      return if value.nil?
+    # Validated here, before the request reaches Protocol::Base#request, rather
+    # than only at the RequestFormatter level. Reaching only the formatter's
+    # check means unwinding through Protocol::Base#request, which logs the
+    # failure as unexpected and closes the connection -- a caller passing a
+    # bad token should get a clean ArgumentError and keep its connection.
+    ROUTING_TOKEN_FORBIDDEN = /[\r\n\0]/
+    private_constant :ROUTING_TOKEN_FORBIDDEN
+
+    def validate_routing_tokens!(req_options)
+      return unless req_options.is_a?(Hash)
+
+      validate_routing_token!(:p_token, req_options[:p_token])
+      validate_routing_token!(:l_token, req_options[:l_token])
+    end
 
-      value.is_a?(String) ? Integer(value, 10) : Integer(value)
-    rescue ArgumentError, TypeError, FloatDomainError
-      raise ArgumentError, "#{name} must be an Integer, got #{value.inspect}"
+    def validate_routing_token!(name, value)
+      # Only an empty *String* is a no-op; see the matching comment in
+      # RequestFormatter#routing_tokens for why respond_to?(:empty?) is wrong
+      # here (it would also excuse [] / {} from the type check below).
+      return if value.nil? || (value.is_a?(String) && value.empty?)
+      raise ArgumentError, "#{name} must be a String, got #{value.class}" unless value.is_a?(String)
+      raise ArgumentError, "#{name} must not contain CRLF or null bytes" if value.match?(ROUTING_TOKEN_FORBIDDEN)
     end
 
     def cas_core(key, always_set, ttl = nil, req_options = nil)
-      (value, cas) = perform(:cas, key)
+      validate_routing_tokens!(req_options)
+      (value, cas) = perform(:cas, key, req_options)
       return if value.nil? && !always_set
 
       newvalue = yield(value)
@@ -648,7 +831,13 @@ module Dalli
 
     def fetch_with_lock_request(key, ttl, lock_ttl, recache_threshold, req_options)
       server = ring.server_for_key(key)
-      result = server.request(:meta_get, key, { vivify_ttl: lock_ttl, recache_ttl: recache_threshold })
+      # req_options is the base, not the override: fetch_with_lock's own
+      # lock_ttl/recache_threshold parameters must always win, even if a
+      # caller's req_options happened to contain :vivify_ttl/:recache_ttl.
+      meta_options = req_options.is_a?(Hash) ? req_options.dup : {}
+      meta_options[:vivify_ttl] = lock_ttl
+      meta_options[:recache_ttl] = recache_threshold
+      result = server.request(:meta_get, key, meta_options)
 
       return result[:value] unless result[:won_recache]
 

data/lib/dalli/pipelined_deleter.rb

@@ -15,17 +15,24 @@ module Dalli
     ##
     # Deletes multiple keys from memcached.
     #
+    # `req_options` is applied to every delete in the batch (see
+    # Dalli::Client#delete for the supported meta-delete keys).
+    #
     # @param keys [Array<String>] keys to delete
-    # @return [Integer] the number of keys that were deleted. This is
-    #   best-effort: on a network error the operation is retried, and keys
-    #   deleted before the error are not recounted, so the result may
-    #   under-report the number actually removed when a failure occurs.
+    # @return [Integer] the number of keys the server found and acted on. Only
+    #   a key that did not exist decrements this count, so with
+    #   `req_options: {invalidate: true}` it reports how many keys were
+    #   tombstoned rather than removed. This is best-effort: a transient
+    #   network error is retried automatically, and keys handled before the
+    #   error are not recounted, so the result may under-report when a retry
+    #   occurs. If a server remains unreachable after retrying, raises
+    #   Dalli::NetworkError.
     ##
-    def process(keys)
+    def process(keys, req_options = nil)
       return 0 if keys.empty?
 
       @ring.lock do
-        groups = setup_requests(keys)
+        groups = setup_requests(keys, req_options)
         finish_requests(groups)
       end
     rescue Dalli::RetryableNetworkError => e
@@ -36,9 +43,9 @@ module Dalli
 
     private
 
-    def setup_requests(keys)
+    def setup_requests(keys, req_options = nil)
       groups = groups_for_keys(keys)
-      make_delete_requests(groups)
+      make_delete_requests(groups, req_options)
       groups
     end
 
@@ -46,12 +53,20 @@ module Dalli
     # Loop through the server-grouped sets of keys, writing
     # the corresponding quiet delete requests to the appropriate servers
     ##
-    def make_delete_requests(groups)
+    # NetworkError (which RetryableNetworkError subclasses) must propagate: the
+    # top-level rescue in #process retries the whole pipelined delete on it. A
+    # combined `rescue DalliError, NetworkError` would silently swallow
+    # RetryableNetworkError too -- since NetworkError < DalliError -- dropping
+    # this server's keys on a transient hiccup instead of retrying. Only a
+    # non-network DalliError should be swallowed here.
+    def make_delete_requests(groups, req_options = nil)
       groups.each do |server, keys_for_server|
         keys_for_server.select! do |key|
-          server.request(:pipelined_delete, key)
+          server.request(:pipelined_delete, key, req_options)
           true
-        rescue DalliError, NetworkError => e
+        rescue Dalli::NetworkError
+          raise
+        rescue DalliError => e
           Dalli.logger.debug { e.inspect }
           Dalli.logger.debug { "unable to delete key #{key} for server #{server.name}" }
           false
@@ -66,7 +81,9 @@ module Dalli
     def finish_requests(groups)
       groups.sum do |server, keys_for_server|
         server.request(:finish_pipelined_delete, keys_for_server.size)
-      rescue DalliError, NetworkError => e
+      rescue Dalli::NetworkError
+        raise
+      rescue DalliError => e
         Dalli.logger.debug { e.inspect }
         Dalli.logger.debug { "unable to complete pipelined delete on server #{server.name}" }
         0