dalli 5.0.6 → 5.1.0

data/lib/dalli/pipelined_deleter.rb

@@ -15,17 +15,24 @@ module Dalli
     ##
     # Deletes multiple keys from memcached.
     #
+    # `req_options` is applied to every delete in the batch (see
+    # Dalli::Client#delete for the supported meta-delete keys).
+    #
     # @param keys [Array<String>] keys to delete
-    # @return [Integer] the number of keys that were deleted. This is
-    #   best-effort: on a network error the operation is retried, and keys
-    #   deleted before the error are not recounted, so the result may
-    #   under-report the number actually removed when a failure occurs.
+    # @return [Integer] the number of keys the server found and acted on. Only
+    #   a key that did not exist decrements this count, so with
+    #   `req_options: {invalidate: true}` it reports how many keys were
+    #   tombstoned rather than removed. This is best-effort: a transient
+    #   network error is retried automatically, and keys handled before the
+    #   error are not recounted, so the result may under-report when a retry
+    #   occurs. If a server remains unreachable after retrying, raises
+    #   Dalli::NetworkError.
     ##
-    def process(keys)
+    def process(keys, req_options = nil)
       return 0 if keys.empty?
 
       @ring.lock do
-        groups = setup_requests(keys)
+        groups = setup_requests(keys, req_options)
         finish_requests(groups)
       end
     rescue Dalli::RetryableNetworkError => e
@@ -36,9 +43,9 @@ module Dalli
 
     private
 
-    def setup_requests(keys)
+    def setup_requests(keys, req_options = nil)
       groups = groups_for_keys(keys)
-      make_delete_requests(groups)
+      make_delete_requests(groups, req_options)
       groups
     end
 
@@ -46,12 +53,20 @@ module Dalli
     # Loop through the server-grouped sets of keys, writing
     # the corresponding quiet delete requests to the appropriate servers
     ##
-    def make_delete_requests(groups)
+    # NetworkError (which RetryableNetworkError subclasses) must propagate: the
+    # top-level rescue in #process retries the whole pipelined delete on it. A
+    # combined `rescue DalliError, NetworkError` would silently swallow
+    # RetryableNetworkError too -- since NetworkError < DalliError -- dropping
+    # this server's keys on a transient hiccup instead of retrying. Only a
+    # non-network DalliError should be swallowed here.
+    def make_delete_requests(groups, req_options = nil)
       groups.each do |server, keys_for_server|
         keys_for_server.select! do |key|
-          server.request(:pipelined_delete, key)
+          server.request(:pipelined_delete, key, req_options)
           true
-        rescue DalliError, NetworkError => e
+        rescue Dalli::NetworkError
+          raise
+        rescue DalliError => e
           Dalli.logger.debug { e.inspect }
           Dalli.logger.debug { "unable to delete key #{key} for server #{server.name}" }
           false
@@ -66,7 +81,9 @@ module Dalli
     def finish_requests(groups)
       groups.sum do |server, keys_for_server|
         server.request(:finish_pipelined_delete, keys_for_server.size)
-      rescue DalliError, NetworkError => e
+      rescue Dalli::NetworkError
+        raise
+      rescue DalliError => e
         Dalli.logger.debug { e.inspect }
         Dalli.logger.debug { "unable to complete pipelined delete on server #{server.name}" }
         0

data/lib/dalli/pipelined_getter.rb

@@ -20,13 +20,18 @@ module Dalli
     ##
     # Yields, one at a time, keys and their values+attributes.
     #
-    def process(keys, &block)
+    # `req_options` accepts :p_token/:l_token, applied to every key in the batch.
+    #
+    # A transient network error is retried automatically. If a server remains
+    # unreachable after retrying, raises Dalli::NetworkError.
+    #
+    def process(keys, req_options = nil, &block)
       return {} if keys.empty?
 
       @ring.lock do
         # Stores partial results collected during interleaved send phase
         @partial_results = {}
-        servers = setup_requests(keys)
+        servers = setup_requests(keys, req_options)
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
         # First yield any partial results collected during interleaved send
@@ -40,6 +45,40 @@ module Dalli
       retry
     end
 
+    ##
+    # Stale-aware bulk get across servers.  Returns { key => metadata Hash } for
+    # the keys that were found; see Protocol::Meta#read_multi_with_metadata_req
+    # for why misses are absent rather than present with miss: true.
+    #
+    # Unlike #process this issues one request per server and reads its full
+    # response before moving on, rather than pipelining across servers: the
+    # metadata path has no interleaving support, and the stale-aware callers it
+    # serves fetch far smaller batches than get_multi does.
+    #
+    # `req_options` accepts :p_token/:l_token, applied to every key in the batch.
+    #
+    def process_with_metadata(keys, req_options = nil)
+      return {} if keys.empty?
+
+      @ring.lock do
+        results = {}
+        groups_for_keys(keys).each do |server, keys_for_server|
+          results.merge!(server.request(:read_multi_with_metadata_req, keys_for_server, req_options))
+        rescue Dalli::RetryableNetworkError
+          raise
+        rescue DalliError, NetworkError => e
+          Dalli.logger.debug { e.inspect }
+          Dalli.logger.debug { "unable to get keys for server #{server.name}" }
+        end
+        results.transform_keys! { |key| @key_manager.key_without_namespace(key) }
+        results
+      end
+    rescue Dalli::RetryableNetworkError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { 'retrying pipelined get with metadata because of network error' }
+      retry
+    end
+
     private
 
     def yield_partial_results
@@ -49,9 +88,9 @@ module Dalli
       @partial_results.clear
     end
 
-    def setup_requests(keys)
+    def setup_requests(keys, req_options = nil)
       groups = groups_for_keys(keys)
-      make_getkq_requests(groups)
+      make_getkq_requests(groups, req_options)
 
       # TODO: How does this exit on a NetworkError
       finish_queries(groups.keys)
@@ -65,17 +104,27 @@ module Dalli
     # on the wire by switching from getkq to getq, and using
     # the opaque value to match requests to responses.
     ##
-    def make_getkq_requests(groups)
+    def make_getkq_requests(groups, req_options = nil)
       groups.each do |server, keys_for_server|
         if keys_for_server.size <= INTERLEAVE_THRESHOLD
           # Small batch - send all at once (existing behavior)
-          server.request(:pipelined_get, keys_for_server)
+          server.request(:pipelined_get, keys_for_server, req_options)
         else
           # Large batch - interleave sends with response draining
           # Pass @partial_results directly to avoid hash allocation/merge overhead
-          server.request(:pipelined_get_interleaved, keys_for_server, CHUNK_SIZE, @partial_results)
+          server.request(:pipelined_get_interleaved, keys_for_server, CHUNK_SIZE, @partial_results, req_options)
         end
-      rescue DalliError, NetworkError => e
+      # NetworkError (which RetryableNetworkError subclasses) must propagate:
+      # #process's top-level rescue retries the whole pipelined get on it. This
+      # rescue used to catch DalliError and NetworkError together -- since
+      # NetworkError < DalliError, that silently swallowed RetryableNetworkError
+      # too, dropping this server's keys from the result on a transient hiccup
+      # instead of retrying, with nothing surfaced above debug-level logging.
+      # Only a non-network DalliError (this server genuinely can't serve these
+      # keys) should be swallowed here.
+      rescue Dalli::NetworkError
+        raise
+      rescue DalliError => e
         Dalli.logger.debug { e.inspect }
         Dalli.logger.debug { "unable to get keys for server #{server.name}" }
       end

data/lib/dalli/pipelined_setter.rb

@@ -14,7 +14,9 @@ module Dalli
 
     ##
     # Writes multiple key-value pairs to memcached.
-    # Raises an error if any server is unavailable.
+    #
+    # A transient network error is retried automatically. If a server remains
+    # unreachable after retrying, raises Dalli::NetworkError.
     #
     # @param hash [Hash] key-value pairs to set
     # @param ttl [Integer] time-to-live in seconds
@@ -46,13 +48,21 @@ module Dalli
     # Loop through the server-grouped sets of keys, writing
     # the corresponding quiet set requests to the appropriate servers
     ##
+    # NetworkError (which RetryableNetworkError subclasses) must propagate: the
+    # top-level rescue in #process retries the whole pipelined set on it. A
+    # combined `rescue DalliError, NetworkError` would silently swallow
+    # RetryableNetworkError too -- since NetworkError < DalliError -- dropping
+    # this server's keys on a transient hiccup instead of retrying. Only a
+    # non-network DalliError should be swallowed here.
     def make_set_requests(groups, hash, ttl, req_options)
       groups.each do |server, keys_for_server|
         keys_for_server.each do |key|
           original_key = @key_manager.key_without_namespace(key)
           value = hash[original_key]
           server.request(:pipelined_set, key, value, ttl, req_options)
-        rescue DalliError, NetworkError => e
+        rescue Dalli::NetworkError
+          raise
+        rescue DalliError => e
           Dalli.logger.debug { e.inspect }
           Dalli.logger.debug { "unable to set key #{key} for server #{server.name}" }
         end
@@ -65,7 +75,9 @@ module Dalli
     def finish_requests(servers)
       servers.each do |server|
         server.request(:noop)
-      rescue DalliError, NetworkError => e
+      rescue Dalli::NetworkError
+        raise
+      rescue DalliError => e
         Dalli.logger.debug { e.inspect }
         Dalli.logger.debug { "unable to complete pipelined set on server #{server.name}" }
       end

data/lib/dalli/protocol/base.rb

@@ -78,6 +78,19 @@ module Dalli
       # particular memcached instance is available for use.
       def alive?
         ensure_connected!
+      rescue Dalli::RetryableNetworkError => e
+        # A single connection attempt failure is retryable -- the same
+        # contract #request enforces on the send/receive path. Retrying here
+        # lets error_on_request!'s own fail-count threshold decide when to
+        # give up: it keeps raising RetryableNetworkError until
+        # socket_max_failures is reached, then raises a terminal
+        # NetworkError via down!, which the next rescue converts to false.
+        # Without this, a single transient hiccup during the liveness check
+        # itself -- as opposed to an actual request -- would report this
+        # server as not alive even though it would have reconnected fine.
+        Dalli.logger.debug { e.inspect }
+        Dalli.logger.debug { "retrying connection attempt to #{name} because of network error" }
+        retry
       rescue Dalli::NetworkError
         # ensure_connected! raises a NetworkError if connection fails.  We
         # want to capture that error and convert it to a boolean value here.
@@ -237,13 +250,44 @@ module Dalli
         opts[:cache_nils] ? true : false
       end
 
+      # Extracts opaque routing-token kwargs (:p_token, :l_token) from a
+      # request-options Hash so they can be splatted into a RequestFormatter
+      # call. Returns {} when neither is set, so the splat is a no-op on the
+      # common path. Validation (type, forbidden bytes) happens at the
+      # wire-formatter level, where it can raise uniformly regardless of how
+      # the token reached the formatter.
+      def routing_token_kwargs(opts)
+        return {} unless opts.is_a?(Hash)
+        return {} unless opts[:p_token] || opts[:l_token]
+
+        { p_token: opts[:p_token], l_token: opts[:l_token] }
+      end
+
+      # Maps the client-facing meta-delete options onto RequestFormatter's flag
+      # names, so they can be splatted into a meta_delete call.  Returns {} when
+      # none are set, keeping the splat a no-op on the common path.
+      #
+      # :tombstone_ttl becomes the T flag, which is the same TTL the formatter
+      # already accepted -- deliberately not a second TTL parameter, since two
+      # would allow emitting two T tokens in one request.  It is sanitized like
+      # every other TTL Dalli sends.
+      def tombstone_kwargs(opts)
+        return {} unless opts.is_a?(Hash)
+
+        kwargs = {}
+        kwargs[:stale] = true if opts[:invalidate]
+        kwargs[:ttl] = TtlSanitizer.sanitize(Integer(opts[:tombstone_ttl])) if opts[:tombstone_ttl]
+        kwargs[:drop_value] = true if opts[:drop_value]
+        kwargs
+      end
+
       def connect
         @connection_manager.establish_connection
         @version = version
         up!
       end
 
-      def pipelined_get(keys)
+      def pipelined_get(keys, options = nil)
         # Clear buffer to remove any stale data from interrupted operations.
         # Use clear (not reset) to keep pipeline_complete? = true, which is
         # the expected state before pipeline_response_setup is called.
@@ -251,7 +295,7 @@ module Dalli
 
         req = +''
         keys.each do |key|
-          req << quiet_get_request(key)
+          req << quiet_get_request(key, options)
         end
         # Could send noop here instead of in pipeline_response_setup
         write(req)
@@ -260,7 +304,7 @@ module Dalli
       # For large batches, interleave writing requests with draining responses.
       # This prevents socket buffer deadlock when sending many keys.
       # Populates the provided results hash with any responses drained during send.
-      def pipelined_get_interleaved(keys, chunk_size, results)
+      def pipelined_get_interleaved(keys, chunk_size, results, options = nil)
         # Initialize the response buffer for draining during send phase
         response_buffer.ensure_ready
 
@@ -268,7 +312,7 @@ module Dalli
           # Build and write this chunk of requests
           req = +''
           chunk.each do |key|
-            req << quiet_get_request(key)
+            req << quiet_get_request(key, options)
           end
           write(req)
           @connection_manager.flush

data/lib/dalli/protocol/key_regularizer.rb

@@ -12,8 +12,25 @@ module Dalli
       module KeyRegularizer
         module_function
 
+        # protocol.txt requires that a key "must not include control
+        # characters or whitespace" -- \p{Cntrl} is C0 (0x00-0x1F) plus DEL
+        # (0x7F). \s alone misses NUL and the rest of that range: a key
+        # containing one of those bytes but no whitespace is ASCII-only, so
+        # it would otherwise be written to the wire unencoded. Not a
+        # protocol-injection risk (the text protocol splits on CRLF, not
+        # other control bytes), but a downstream consumer that treats the key
+        # specially at one of those bytes (a C string terminating at NUL, a
+        # terminal or log line interpreting an escape byte) could silently
+        # act on a different key than Dalli believes it sent.
+        #
+        # Written as \p{Cntrl} rather than the POSIX [:cntrl:] bracket class:
+        # \s and [:cntrl:] overlap (tab, newline, CR are in both), and Ruby
+        # warns "character class has duplicated range" when they're combined
+        # in one -- fatal here, since this suite's -w run treats warnings as
+        # errors (see test_strict_warnings.rb). \p{Cntrl} matches the same
+        # bytes without the overlap warning.
         def required?(key)
-          !key.ascii_only? || /\s/.match?(key)
+          !key.ascii_only? || /[\p{Cntrl}\s]/.match?(key)
         end
 
         def encode(key)

data/lib/dalli/protocol/meta.rb

@@ -26,20 +26,21 @@ module Dalli
       def get(key, options = nil)
         # Skip bitflags in raw mode - saves 2 bytes per request and skips parsing
         skip_flags = raw_mode? || (options && options[:raw])
-        req = RequestFormatter.meta_get(key: key, skip_flags: skip_flags)
+        req = RequestFormatter.meta_get(key: key, skip_flags: skip_flags, **routing_token_kwargs(options))
         flushed_write(req)
         response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
       end
 
-      def quiet_get_request(key)
+      def quiet_get_request(key, options = nil)
         # Skip bitflags in raw mode - saves 2 bytes per request and skips parsing
-        RequestFormatter.meta_get(key: key, return_cas: true, quiet: true, skip_flags: raw_mode?)
+        RequestFormatter.meta_get(key: key, return_cas: true, quiet: true, skip_flags: raw_mode?,
+                                  **routing_token_kwargs(options))
       end
 
       def gat(key, ttl, options = nil)
         ttl = TtlSanitizer.sanitize(ttl)
         skip_flags = raw_mode? || (options && options[:raw])
-        req = RequestFormatter.meta_get(key: key, ttl: ttl, skip_flags: skip_flags)
+        req = RequestFormatter.meta_get(key: key, ttl: ttl, skip_flags: skip_flags, **routing_token_kwargs(options))
         flushed_write(req)
         response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
       end
@@ -53,8 +54,8 @@ module Dalli
 
       # TODO: This is confusing, as there's a cas command in memcached
       # and this isn't it.  Maybe rename?  Maybe eliminate?
-      def cas(key)
-        req = RequestFormatter.meta_get(key: key, value: true, return_cas: true)
+      def cas(key, options = nil)
+        req = RequestFormatter.meta_get(key: key, value: true, return_cas: true, **routing_token_kwargs(options))
         flushed_write(req)
         response_processor.meta_get_with_value_and_cas
       end
@@ -88,12 +89,16 @@ module Dalli
           key: key, value: true, return_cas: true,
           vivify_ttl: options[:vivify_ttl], recache_ttl: options[:recache_ttl],
           return_hit_status: options[:return_hit_status],
-          return_last_access: options[:return_last_access], skip_lru_bump: options[:skip_lru_bump]
+          return_last_access: options[:return_last_access],
+          return_ttl_remaining: options[:return_ttl_remaining],
+          skip_lru_bump: options[:skip_lru_bump],
+          **routing_token_kwargs(options)
         )
         flushed_write(req)
         response_processor.meta_get_with_metadata(
           cache_nils: cache_nils?(options), return_hit_status: options[:return_hit_status],
-          return_last_access: options[:return_last_access]
+          return_last_access: options[:return_last_access],
+          return_ttl_remaining: options[:return_ttl_remaining]
         )
       end
 
@@ -137,35 +142,41 @@ module Dalli
         ttl = TtlSanitizer.sanitize(ttl) if ttl
         req = RequestFormatter.meta_set(key: key, value: value,
                                         bitflags: bitflags, cas: cas,
-                                        ttl: ttl, mode: mode, quiet: quiet)
+                                        ttl: ttl, mode: mode, quiet: quiet,
+                                        **routing_token_kwargs(options))
         write("#{req}#{value}#{TERMINATOR}")
         @connection_manager.flush unless quiet
       end
       # rubocop:enable Metrics/ParameterLists
 
-      def append(key, value)
-        write_append_prepend_req(:append, key, value)
+      def append(key, value, options = nil)
+        write_append_prepend_req(:append, key, value, nil, nil, options)
         response_processor.meta_set_append_prepend unless quiet?
       end
 
-      def prepend(key, value)
-        write_append_prepend_req(:prepend, key, value)
+      def prepend(key, value, options = nil)
+        write_append_prepend_req(:prepend, key, value, nil, nil, options)
         response_processor.meta_set_append_prepend unless quiet?
       end
 
       # rubocop:disable Metrics/ParameterLists
-      def write_append_prepend_req(mode, key, value, ttl = nil, cas = nil, _options = {})
+      def write_append_prepend_req(mode, key, value, ttl = nil, cas = nil, options = nil)
         ttl = TtlSanitizer.sanitize(ttl) if ttl
         req = RequestFormatter.meta_set(key: key, value: value,
-                                        cas: cas, ttl: ttl, mode: mode, quiet: quiet?)
+                                        cas: cas, ttl: ttl, mode: mode, quiet: quiet?,
+                                        **routing_token_kwargs(options))
         write("#{req}#{value}#{TERMINATOR}")
         @connection_manager.flush unless quiet?
       end
       # rubocop:enable Metrics/ParameterLists
 
       # Delete Commands
-      def delete(key, cas)
-        req = RequestFormatter.meta_delete(key: key, cas: cas, quiet: quiet?)
+      #
+      # `options` supports the meta-delete keys :invalidate, :tombstone_ttl and
+      # :drop_value, plus :p_token/:l_token; see Dalli::Client#delete.
+      def delete(key, cas, options = nil)
+        req = RequestFormatter.meta_delete(key: key, cas: cas, quiet: quiet?,
+                                           **tombstone_kwargs(options), **routing_token_kwargs(options))
         write(req)
         @connection_manager.flush unless quiet?
         response_processor.meta_delete unless quiet?
@@ -173,8 +184,9 @@ module Dalli
 
       # Pipelined delete - writes a quiet delete request without reading response.
       # Used by PipelinedDeleter for bulk operations.
-      def pipelined_delete(key)
-        req = RequestFormatter.meta_delete(key: key, quiet: true)
+      def pipelined_delete(key, req_options = nil)
+        req = RequestFormatter.meta_delete(key: key, quiet: true,
+                                           **tombstone_kwargs(req_options), **routing_token_kwargs(req_options))
         write(req)
       end
 
@@ -184,21 +196,23 @@ module Dalli
       end
 
       # Arithmetic Commands
-      def decr(key, count, ttl, initial)
-        decr_incr false, key, count, ttl, initial
+      def decr(key, count, ttl, initial, options = nil)
+        decr_incr false, key, count, ttl, initial, options
       end
 
-      def incr(key, count, ttl, initial)
-        decr_incr true, key, count, ttl, initial
+      def incr(key, count, ttl, initial, options = nil)
+        decr_incr true, key, count, ttl, initial, options
       end
 
-      def decr_incr(incr, key, delta, ttl, initial)
+      # rubocop:disable Metrics/ParameterLists
+      def decr_incr(incr, key, delta, ttl, initial, options = nil)
         ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
         write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: initial, incr: incr, ttl: ttl,
-                                               quiet: quiet?))
+                                               quiet: quiet?, **routing_token_kwargs(options)))
         @connection_manager.flush unless quiet?
         response_processor.decr_incr unless quiet?
       end
+      # rubocop:enable Metrics/ParameterLists
 
       # Other Commands
       def flush(delay = 0)
@@ -236,14 +250,52 @@ module Dalli
       # Single-server fast path for get_multi. Inlines request formatting and
       # response parsing to minimize per-key overhead. Avoids the PipelinedGetter
       # machinery (IO.select, response buffering, server grouping).
-      def read_multi_req(keys)
+      def read_multi_req(keys, options = nil)
         is_raw = raw_mode?
-        buffer = RequestFormatter.multi_meta_get(keys, skip_flags: is_raw)
+        buffer = RequestFormatter.multi_meta_get(keys, skip_flags: is_raw, **routing_token_kwargs(options))
         flushed_write(buffer)
         buffer.clear
         read_multi_get_responses(is_raw)
       end
 
+      # Stale-aware bulk get.  Returns { key => { value:, cas:, stale:, miss: } }
+      # for the keys the server returned.  Keys that were not found are absent
+      # from the hash, matching read_multi_req and the get_multi family; a
+      # tombstoned item is present (it answers VA with the X flag) with
+      # stale: true, which is the distinction callers need.
+      #
+      # Shared by both the single-server fast path and PipelinedGetter's
+      # per-server-group request, so this one change covers both routes.
+      def read_multi_with_metadata_req(keys, options = nil)
+        is_raw = raw_mode?
+        buffer = RequestFormatter.multi_meta_get(keys, skip_flags: is_raw, return_cas: true,
+                                                       **routing_token_kwargs(options))
+        flushed_write(buffer)
+        buffer.clear
+        read_multi_metadata_responses(is_raw)
+      end
+
+      # Unlike read_multi_get_responses this locates tokens by flag rather than
+      # by position, because the c flag shifts the key's index.
+      def read_multi_metadata_responses(is_raw)
+        hash = {}
+        while (line = @connection_manager.read_line)
+          break if line.start_with?('MN')
+          next unless line.start_with?('VA ')
+
+          tokens = line.chomp!(TERMINATOR).split
+          value = @connection_manager.read(tokens[1].to_i + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
+          stale = response_processor.stale_from_tokens(tokens)
+          cas = response_processor.cas_from_tokens(tokens)
+          bitflags = is_raw ? 0 : response_processor.bitflags_from_tokens(tokens)
+          key = response_processor.key_from_tokens(tokens)
+          next if key.nil?
+
+          hash[key] = { value: @value_marshaller.retrieve(value, bitflags), cas: cas, stale: stale, miss: false }
+        end
+        hash
+      end
+
       def read_multi_get_responses(is_raw)
         hash = {}
         key_index = is_raw ? 2 : 3
@@ -277,7 +329,7 @@ module Dalli
           [key, @value_marshaller.store(key, raw_value, req_options)]
         end
 
-        buffer = RequestFormatter.multi_meta_set(entries, ttl: ttl)
+        buffer = RequestFormatter.multi_meta_set(entries, ttl: ttl, **routing_token_kwargs(req_options))
         flushed_write(buffer)
         buffer.clear
         response_processor.consume_all_responses_until_mn
@@ -285,8 +337,9 @@ module Dalli
 
       # Single-server fast path for delete_multi. Writes all quiet delete requests
       # terminated by a noop, then consumes all responses.
-      def delete_multi_req(keys)
-        buffer = RequestFormatter.multi_meta_delete(keys)
+      def delete_multi_req(keys, req_options = nil)
+        buffer = RequestFormatter.multi_meta_delete(keys, **tombstone_kwargs(req_options),
+                                                    **routing_token_kwargs(req_options))
         flushed_write(buffer)
         buffer.clear
         keys.size - response_processor.pipelined_delete_non_deletions